2024-11-25 02:13:59,681 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b7a7f33
2024-11-25 02:13:59,747 main DEBUG Took 0.053209 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-25 02:13:59,747 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-25 02:13:59,748 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-25 02:13:59,749 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-25 02:13:59,751 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 02:13:59,769 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-25 02:13:59,798 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 02:13:59,803 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 02:13:59,804 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 02:13:59,804 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 02:13:59,805 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 02:13:59,805 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 02:13:59,807 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 02:13:59,807 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 02:13:59,808 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 02:13:59,809 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 02:13:59,810 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 02:13:59,810 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 02:13:59,811 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 02:13:59,812 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 02:13:59,813 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 02:13:59,813 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 02:13:59,814 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 02:13:59,814 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 02:13:59,815 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 02:13:59,817 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 02:13:59,818 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 02:13:59,818 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 02:13:59,819 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 02:13:59,820 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 02:13:59,821 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 02:13:59,821 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-25 02:13:59,824 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 02:13:59,826 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-25 02:13:59,829 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-25 02:13:59,829 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-25 02:13:59,831 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-25 02:13:59,832 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-25 02:13:59,854 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-25 02:13:59,858 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-25 02:13:59,862 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-25 02:13:59,863 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-25 02:13:59,864 main DEBUG createAppenders(={Console})
2024-11-25 02:13:59,865 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b7a7f33 initialized
2024-11-25 02:13:59,866 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b7a7f33
2024-11-25 02:13:59,866 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b7a7f33 OK.
2024-11-25 02:13:59,867 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-25 02:13:59,867 main DEBUG OutputStream closed
2024-11-25 02:13:59,868 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-25 02:13:59,869 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-25 02:13:59,869 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@485966cc OK
2024-11-25 02:14:00,041 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-25 02:14:00,047 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-25 02:14:00,048 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-25 02:14:00,050 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-25 02:14:00,051 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-25 02:14:00,051 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-25 02:14:00,061 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-25 02:14:00,061 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-25 02:14:00,062 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-25 02:14:00,062 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-25 02:14:00,063 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-25 02:14:00,063 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-25 02:14:00,066 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-25 02:14:00,067 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-25 02:14:00,067 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-25 02:14:00,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-25 02:14:00,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-25 02:14:00,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-25 02:14:00,073 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-25 02:14:00,074 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4426bff1) with optional ClassLoader: null
2024-11-25 02:14:00,074 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-25 02:14:00,075 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4426bff1] started OK.
2024-11-25T02:14:00,528 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e86d295f-9341-d83f-1603-cbf4e060f84f
2024-11-25 02:14:00,533 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-25 02:14:00,534 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-25T02:14:00,574 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.security.TestSecureIPC timeout: 52 mins
2024-11-25T02:14:00,938 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=13, OpenFileDescriptor=288, MaxFileDescriptor=1048576, SystemLoadAverage=660, ProcessCount=11, AvailableMemoryMB=4583
2024-11-25T02:14:01,181 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-25T02:14:01,313 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:01,322 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:01,432 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500841427,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:02,733 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:02,882 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-25T02:14:02,952 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-25T02:14:02,979 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-25T02:14:03,058 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39493
2024-11-25T02:14:03,100 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:03,105 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:03,108 INFO  [Listener,port=39493 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39493: starting
2024-11-25T02:14:03,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a476bc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:03,348 DEBUG [Thread-6 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:03,459 DEBUG [Thread-6 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39493
2024-11-25T02:14:03,506 DEBUG [Thread-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:03,616 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500843614,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:03,627 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:03,632 DEBUG [Reader=1,bindAddress=localhost,port=39493 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:03,638 DEBUG [Reader=1,bindAddress=localhost,port=39493 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:03,639 DEBUG [Reader=1,bindAddress=localhost,port=39493 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:03,660 DEBUG [Reader=1,bindAddress=localhost,port=39493 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:03,664 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:03,666 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:03,672 DEBUG [Reader=1,bindAddress=localhost,port=39493 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:03,680 DEBUG [Reader=1,bindAddress=localhost,port=39493 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:03,682 DEBUG [Reader=1,bindAddress=localhost,port=39493 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:03,683 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:03,688 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:03,689 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-25T02:14:03,690 DEBUG [Thread-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:03,692 DEBUG [Reader=1,bindAddress=localhost,port=39493 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:03,693 DEBUG [Reader=1,bindAddress=localhost,port=39493 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:03,694 DEBUG [Reader=1,bindAddress=localhost,port=39493 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:03,694 DEBUG [Reader=1,bindAddress=localhost,port=39493 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T02:14:03,695 INFO  [Reader=1,bindAddress=localhost,port=39493 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:03,723 DEBUG [Thread-6 {}] ipc.BlockingRpcConnection(634): Length of response for connection header:93
2024-11-25T02:14:03,732 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:242
2024-11-25T02:14:03,734 INFO  [Reader=1,bindAddress=localhost,port=39493 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57322, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:03,735 DEBUG [BRPC Connection (1107543386) to localhost:39493 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-25T02:14:03,789 DEBUG [BRPC Connection (1107543386) to localhost:39493 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:228
2024-11-25T02:14:03,796 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:2271
2024-11-25T02:14:03,799 DEBUG [BRPC Connection (1107543386) to localhost:39493 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-25T02:14:03,804 DEBUG [BRPC Connection (1107543386) to localhost:39493 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:2257
2024-11-25T02:14:03,830 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:4117
2024-11-25T02:14:03,833 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:18171
2024-11-25T02:14:03,841 DEBUG [BRPC Connection (1107543386) to localhost:39493 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-25T02:14:03,862 DEBUG [BRPC Connection (1107543386) to localhost:39493 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:22274
2024-11-25T02:14:03,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:03,871 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39493
2024-11-25T02:14:03,871 INFO  [Listener,port=39493 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39493: stopping
2024-11-25T02:14:03,871 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:03,872 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:03,880 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 13)
Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: pool-3-thread-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 288) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 660) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4203 (was 4583)
2024-11-25T02:14:03,891 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=696, ProcessCount=11, AvailableMemoryMB=4203
2024-11-25T02:14:03,917 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:03,917 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:03,927 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500843926,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:03,948 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:03,953 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39833
2024-11-25T02:14:03,956 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:03,957 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:03,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26e0680c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null
2024-11-25T02:14:03,959 INFO  [Listener,port=39833 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39833: starting
2024-11-25T02:14:03,960 DEBUG [Thread-11 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:03,960 DEBUG [Thread-11 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39833
2024-11-25T02:14:03,962 DEBUG [Thread-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:03,979 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500843977,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:03,984 DEBUG [Thread-11 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:03,984 DEBUG [Thread-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T02:14:03,986 DEBUG [Thread-11 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:39833
org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections.
	at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:111) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-25T02:14:03,991 WARN  [Thread-11 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732500843931
2024-11-25T02:14:04,063 DEBUG [Thread-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,067 DEBUG [Thread-11 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:04,068 DEBUG [Thread-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T02:14:04,072 DEBUG [Thread-11 {}] ipc.FailedServers(52): Added failed server with address localhost:39833 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,073 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:04,073 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39833
2024-11-25T02:14:04,074 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:04,075 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:04,088 INFO  [Listener,port=39833 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39833: stopping
2024-11-25T02:14:04,094 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=25 (was 19)
Potentially hanging thread: Reader=1,bindAddress=localhost,port=39833
	java.base@17.0.11/jdk.internal.misc.Unsafe.getInt(Native Method)
	java.base@17.0.11/jdk.internal.misc.Unsafe.getInt(Unsafe.java:364)
	java.base@17.0.11/sun.nio.ch.EPoll.getDescriptor(EPoll.java:94)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.processEvents(EPollSelectorImpl.java:188)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:135)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer idle connection scanner for port 39833
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: Listener,port=39833
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:213)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:115)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63)

Potentially hanging thread: Reader=0,bindAddress=localhost,port=39833
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=2,bindAddress=localhost,port=39833
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=293 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 696), ProcessCount=11 (was 11), AvailableMemoryMB=4190 (was 4203)
2024-11-25T02:14:04,112 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=696, ProcessCount=11, AvailableMemoryMB=4190
2024-11-25T02:14:04,132 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:04,133 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:04,141 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500844140,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:04,150 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:04,156 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40301
2024-11-25T02:14:04,176 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:04,177 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:04,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4751adec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:04,179 INFO  [Listener,port=40301 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40301: starting
2024-11-25T02:14:04,185 DEBUG [Thread-14 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:04,186 DEBUG [Thread-14 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40301
2024-11-25T02:14:04,187 DEBUG [Thread-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,199 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500844198,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,203 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:04,204 DEBUG [Reader=1,bindAddress=localhost,port=40301 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,206 DEBUG [Reader=1,bindAddress=localhost,port=40301 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:04,206 DEBUG [Reader=1,bindAddress=localhost,port=40301 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:04,209 DEBUG [Reader=1,bindAddress=localhost,port=40301 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:04,210 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:04,211 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:04,211 DEBUG [Reader=1,bindAddress=localhost,port=40301 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:04,212 DEBUG [Reader=1,bindAddress=localhost,port=40301 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:04,212 DEBUG [Reader=1,bindAddress=localhost,port=40301 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:04,213 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:04,214 DEBUG [Thread-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T02:14:04,214 DEBUG [Thread-14 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:40301
javax.security.sasl.SaslException: No common protection layer between client and server
	at com.sun.security.sasl.gsskerb.GssKrb5Client.doFinalHandshake(GssKrb5Client.java:270) ~[jdk.security.jgss:?]
	at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:203) ~[jdk.security.jgss:?]
	at org.apache.hadoop.hbase.security.AbstractHBaseSaslRpcClient.evaluateChallenge(AbstractHBaseSaslRpcClient.java:105) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:126) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-25T02:14:04,215 WARN  [Thread-14 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732500844146
2024-11-25T02:14:04,236 DEBUG [Thread-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,240 DEBUG [Reader=2,bindAddress=localhost,port=40301 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,242 DEBUG [Reader=2,bindAddress=localhost,port=40301 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:04,242 DEBUG [Reader=2,bindAddress=localhost,port=40301 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:04,246 DEBUG [Reader=2,bindAddress=localhost,port=40301 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:04,246 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:04,246 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:04,247 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:04,248 DEBUG [Reader=2,bindAddress=localhost,port=40301 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:04,248 DEBUG [Reader=2,bindAddress=localhost,port=40301 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:04,248 DEBUG [Reader=2,bindAddress=localhost,port=40301 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:04,249 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:04,249 DEBUG [Thread-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T02:14:04,250 DEBUG [Thread-14 {}] ipc.FailedServers(52): Added failed server with address localhost:40301 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,250 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:04,251 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40301
2024-11-25T02:14:04,254 INFO  [Listener,port=40301 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40301: stopping
2024-11-25T02:14:04,255 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:04,255 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:04,267 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=25 (was 19)
Potentially hanging thread: Reader=1,bindAddress=localhost,port=40301
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:115)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63)

Potentially hanging thread: Reader=0,bindAddress=localhost,port=40301
	java.base@17.0.11/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:265)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:134)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=2,bindAddress=localhost,port=40301
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer idle connection scanner for port 40301
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: Listener,port=40301
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:213)
 - Thread LEAK? -, OpenFileDescriptor=299 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 696), ProcessCount=11 (was 11), AvailableMemoryMB=4171 (was 4190)
2024-11-25T02:14:04,283 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=292, MaxFileDescriptor=1048576, SystemLoadAverage=696, ProcessCount=11, AvailableMemoryMB=4170
2024-11-25T02:14:04,304 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:04,304 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:04,314 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500844313,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:04,321 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:04,340 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40903
2024-11-25T02:14:04,355 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:04,356 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:04,357 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@202d88aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:04,360 INFO  [Listener,port=40903 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40903: starting
2024-11-25T02:14:04,365 DEBUG [Thread-17 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:04,369 DEBUG [Thread-17 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40903
2024-11-25T02:14:04,370 DEBUG [Thread-17 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,396 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500844395,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,400 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:04,401 DEBUG [Reader=1,bindAddress=localhost,port=40903 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,403 DEBUG [Reader=1,bindAddress=localhost,port=40903 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:04,403 DEBUG [Reader=1,bindAddress=localhost,port=40903 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:04,415 DEBUG [Reader=1,bindAddress=localhost,port=40903 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:04,416 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:04,417 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:04,417 DEBUG [Reader=1,bindAddress=localhost,port=40903 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:04,417 DEBUG [Reader=1,bindAddress=localhost,port=40903 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:04,418 DEBUG [Reader=1,bindAddress=localhost,port=40903 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:04,418 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:04,419 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:04,419 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T02:14:04,420 DEBUG [Thread-17 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:04,420 DEBUG [Reader=1,bindAddress=localhost,port=40903 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:04,421 DEBUG [Reader=1,bindAddress=localhost,port=40903 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:04,421 DEBUG [Reader=1,bindAddress=localhost,port=40903 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,421 DEBUG [Reader=1,bindAddress=localhost,port=40903 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:04,422 INFO  [Reader=1,bindAddress=localhost,port=40903 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:04,422 INFO  [Reader=1,bindAddress=localhost,port=40903 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40676, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:04,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:04,501 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40903
2024-11-25T02:14:04,501 INFO  [Listener,port=40903 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40903: stopping
2024-11-25T02:14:04,501 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:04,501 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:04,502 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:04,512 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=36179
2024-11-25T02:14:04,521 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:04,522 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:04,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2319b8fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:04,523 INFO  [Listener,port=36179 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=36179: starting
2024-11-25T02:14:04,526 DEBUG [Thread-20 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:04,527 DEBUG [Thread-20 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:36179
2024-11-25T02:14:04,528 DEBUG [Thread-20 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,532 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:04,536 DEBUG [Reader=1,bindAddress=localhost,port=36179 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,541 DEBUG [Reader=1,bindAddress=localhost,port=36179 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:04,541 DEBUG [Reader=1,bindAddress=localhost,port=36179 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:04,545 DEBUG [Reader=1,bindAddress=localhost,port=36179 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:04,545 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:04,546 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:04,549 DEBUG [Reader=1,bindAddress=localhost,port=36179 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:04,550 DEBUG [Reader=1,bindAddress=localhost,port=36179 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:04,550 DEBUG [Reader=1,bindAddress=localhost,port=36179 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:04,553 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:04,554 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:04,555 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T02:14:04,555 DEBUG [Thread-20 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:04,557 DEBUG [Reader=1,bindAddress=localhost,port=36179 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:04,557 DEBUG [Reader=1,bindAddress=localhost,port=36179 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:04,558 DEBUG [Reader=1,bindAddress=localhost,port=36179 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,558 DEBUG [Reader=1,bindAddress=localhost,port=36179 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:04,558 INFO  [Reader=1,bindAddress=localhost,port=36179 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:04,559 INFO  [Reader=1,bindAddress=localhost,port=36179 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50080, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:04,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:04,609 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 36179
2024-11-25T02:14:04,609 INFO  [Listener,port=36179 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=36179: stopping
2024-11-25T02:14:04,610 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:04,610 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:04,610 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:04,613 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34061
2024-11-25T02:14:04,617 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:04,620 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:04,620 INFO  [Listener,port=34061 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34061: starting
2024-11-25T02:14:04,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@289b966d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:04,623 DEBUG [Thread-23 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:04,624 DEBUG [Thread-23 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34061
2024-11-25T02:14:04,625 DEBUG [Thread-23 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,628 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:04,628 DEBUG [Reader=1,bindAddress=localhost,port=34061 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,630 DEBUG [Reader=1,bindAddress=localhost,port=34061 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:04,630 DEBUG [Reader=1,bindAddress=localhost,port=34061 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:04,634 DEBUG [Reader=1,bindAddress=localhost,port=34061 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:04,637 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:04,638 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:04,639 DEBUG [Reader=1,bindAddress=localhost,port=34061 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:04,639 DEBUG [Reader=1,bindAddress=localhost,port=34061 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:04,640 DEBUG [Reader=1,bindAddress=localhost,port=34061 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:04,640 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:04,641 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:04,641 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T02:14:04,642 DEBUG [Thread-23 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:04,642 DEBUG [Reader=1,bindAddress=localhost,port=34061 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:04,642 DEBUG [Reader=1,bindAddress=localhost,port=34061 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:04,643 DEBUG [Reader=1,bindAddress=localhost,port=34061 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,643 DEBUG [Reader=1,bindAddress=localhost,port=34061 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:04,643 INFO  [Reader=1,bindAddress=localhost,port=34061 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:04,643 INFO  [Reader=1,bindAddress=localhost,port=34061 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60188, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:04,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:04,697 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34061
2024-11-25T02:14:04,697 INFO  [Listener,port=34061 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34061: stopping
2024-11-25T02:14:04,698 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:04,699 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:04,700 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:04,715 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=43869
2024-11-25T02:14:04,716 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:04,720 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:04,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6022ba69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:04,722 INFO  [Listener,port=43869 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=43869: starting
2024-11-25T02:14:04,728 DEBUG [Thread-26 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:04,729 DEBUG [Thread-26 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43869
2024-11-25T02:14:04,731 DEBUG [Thread-26 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,735 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:04,736 DEBUG [Reader=1,bindAddress=localhost,port=43869 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,738 DEBUG [Reader=1,bindAddress=localhost,port=43869 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:04,738 DEBUG [Reader=1,bindAddress=localhost,port=43869 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:04,741 DEBUG [Reader=1,bindAddress=localhost,port=43869 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:04,742 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:04,743 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:04,743 DEBUG [Reader=1,bindAddress=localhost,port=43869 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:04,743 DEBUG [Reader=1,bindAddress=localhost,port=43869 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:04,744 DEBUG [Reader=1,bindAddress=localhost,port=43869 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:04,744 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:04,745 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:04,745 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-int
2024-11-25T02:14:04,745 DEBUG [Reader=1,bindAddress=localhost,port=43869 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:04,745 DEBUG [Thread-26 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:04,746 DEBUG [Reader=1,bindAddress=localhost,port=43869 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:04,746 DEBUG [Reader=1,bindAddress=localhost,port=43869 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,746 DEBUG [Reader=1,bindAddress=localhost,port=43869 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int
2024-11-25T02:14:04,746 INFO  [Reader=1,bindAddress=localhost,port=43869 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:04,747 INFO  [Reader=1,bindAddress=localhost,port=43869 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55944, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:04,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:04,773 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 43869
2024-11-25T02:14:04,773 INFO  [Listener,port=43869 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=43869: stopping
2024-11-25T02:14:04,780 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:04,781 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:04,784 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:04,790 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=35999
2024-11-25T02:14:04,791 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:04,792 INFO  [Listener,port=35999 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=35999: starting
2024-11-25T02:14:04,792 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:04,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@261353df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:04,796 DEBUG [Thread-29 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:04,797 DEBUG [Thread-29 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:35999
2024-11-25T02:14:04,798 DEBUG [Thread-29 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,806 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:04,806 DEBUG [Reader=1,bindAddress=localhost,port=35999 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,807 DEBUG [Reader=1,bindAddress=localhost,port=35999 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:04,807 DEBUG [Reader=1,bindAddress=localhost,port=35999 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:04,809 DEBUG [Reader=1,bindAddress=localhost,port=35999 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:04,810 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:04,810 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:04,810 DEBUG [Reader=1,bindAddress=localhost,port=35999 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:04,810 DEBUG [Reader=1,bindAddress=localhost,port=35999 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:04,811 DEBUG [Reader=1,bindAddress=localhost,port=35999 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:04,811 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:04,812 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:04,812 DEBUG [Reader=1,bindAddress=localhost,port=35999 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:04,812 DEBUG [Reader=1,bindAddress=localhost,port=35999 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:04,812 DEBUG [Reader=1,bindAddress=localhost,port=35999 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:04,812 DEBUG [Reader=1,bindAddress=localhost,port=35999 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T02:14:04,813 INFO  [Reader=1,bindAddress=localhost,port=35999 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:04,813 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-25T02:14:04,813 DEBUG [Thread-29 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:04,814 INFO  [Reader=1,bindAddress=localhost,port=35999 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40142, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:04,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:04,845 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 35999
2024-11-25T02:14:04,849 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:04,849 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:04,852 INFO  [Listener,port=35999 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=35999: stopping
2024-11-25T02:14:04,862 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=24 (was 19)
Potentially hanging thread: Listener,port=35999
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:213)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:115)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63)

Potentially hanging thread: RpcServer idle connection scanner for port 35999
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: Reader=2,bindAddress=localhost,port=35999
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: BRPC Connection (1107543386) to localhost:35999 from hbase/localhost@EXAMPLE.COM
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.hbase.ipc.BlockingRpcConnection.waitForWork(BlockingRpcConnection.java:329)
	app//org.apache.hadoop.hbase.ipc.BlockingRpcConnection.run(BlockingRpcConnection.java:342)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=293 (was 292) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 696), ProcessCount=11 (was 11), AvailableMemoryMB=4154 (was 4170)
2024-11-25T02:14:04,871 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=696, ProcessCount=11, AvailableMemoryMB=4153
2024-11-25T02:14:04,887 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:04,887 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:04,894 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500844893,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:04,899 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:04,906 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=43899
2024-11-25T02:14:04,907 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:04,907 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:04,907 INFO  [Listener,port=43899 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=43899: starting
2024-11-25T02:14:04,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b68d50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:04,909 DEBUG [Thread-32 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-25T02:14:04,910 DEBUG [Thread-32 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43899
2024-11-25T02:14:04,918 DEBUG [BRPC Connection (1107543386) to localhost:43899 from testuser {}] ipc.RpcConnection(424): process preamble call response with response type null
2024-11-25T02:14:04,919 WARN  [BRPC Connection (1107543386) to localhost:43899 from testuser {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call
2024-11-25T02:14:04,921 DEBUG [BRPC Connection (1107543386) to localhost:43899 from testuser {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes
2024-11-25T02:14:04,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:04,928 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 43899
2024-11-25T02:14:04,929 INFO  [Listener,port=43899 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=43899: stopping
2024-11-25T02:14:04,929 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:04,929 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:04,943 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 696), ProcessCount=11 (was 11), AvailableMemoryMB=4144 (was 4153)
2024-11-25T02:14:04,950 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=696, ProcessCount=11, AvailableMemoryMB=4144
2024-11-25T02:14:04,967 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:04,968 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:04,975 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500844974,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:04,979 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:04,988 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=35369
2024-11-25T02:14:04,992 WARN  [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! *********
2024-11-25T02:14:04,992 WARN  [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients
2024-11-25T02:14:04,993 WARN  [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true).
2024-11-25T02:14:04,993 WARN  [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user
2024-11-25T02:14:04,993 WARN  [Time-limited test {}] ipc.RpcServer(380): impersonation is possible!
2024-11-25T02:14:04,993 WARN  [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible,
2024-11-25T02:14:04,993 WARN  [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml
2024-11-25T02:14:04,993 WARN  [Time-limited test {}] ipc.RpcServer(383): ****************************
2024-11-25T02:14:04,994 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:04,996 INFO  [Listener,port=35369 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=35369: starting
2024-11-25T02:14:04,996 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:04,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d77440d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:05,001 DEBUG [Thread-35 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-25T02:14:05,002 DEBUG [Thread-35 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:35369
2024-11-25T02:14:05,004 WARN  [Reader=1,bindAddress=localhost,port=35369 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1
2024-11-25T02:14:05,004 INFO  [Reader=1,bindAddress=localhost,port=35369 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:52140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-25T02:14:05,056 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:05,057 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 35369
2024-11-25T02:14:05,057 INFO  [Listener,port=35369 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=35369: stopping
2024-11-25T02:14:05,057 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:05,058 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:05,076 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 19), OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 696), ProcessCount=11 (was 11), AvailableMemoryMB=4133 (was 4144)
2024-11-25T02:14:05,086 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=696, ProcessCount=11, AvailableMemoryMB=4132
2024-11-25T02:14:05,103 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:05,103 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:05,110 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500845109,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:05,114 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:05,132 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41411
2024-11-25T02:14:05,134 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:05,135 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:05,135 INFO  [Listener,port=41411 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41411: starting
2024-11-25T02:14:05,135 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ce5203e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:05,145 DEBUG [Thread-38 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:05,145 DEBUG [Thread-38 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:41411
2024-11-25T02:14:05,152 DEBUG [Thread-38 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:05,164 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500845163,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:05,167 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:05,169 DEBUG [Reader=1,bindAddress=localhost,port=41411 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:05,170 DEBUG [Reader=1,bindAddress=localhost,port=41411 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:05,170 DEBUG [Reader=1,bindAddress=localhost,port=41411 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:05,173 DEBUG [Reader=1,bindAddress=localhost,port=41411 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:05,173 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:05,174 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:05,174 DEBUG [Reader=1,bindAddress=localhost,port=41411 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:05,175 DEBUG [Reader=1,bindAddress=localhost,port=41411 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:05,179 DEBUG [Reader=1,bindAddress=localhost,port=41411 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:05,180 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:05,180 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:05,181 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T02:14:05,181 DEBUG [Reader=1,bindAddress=localhost,port=41411 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:05,181 DEBUG [Thread-38 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:05,181 DEBUG [Reader=1,bindAddress=localhost,port=41411 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:05,181 DEBUG [Reader=1,bindAddress=localhost,port=41411 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:05,181 DEBUG [Reader=1,bindAddress=localhost,port=41411 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:05,181 INFO  [Reader=1,bindAddress=localhost,port=41411 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:05,182 INFO  [Reader=1,bindAddress=localhost,port=41411 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34326, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:05,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:05,217 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41411
2024-11-25T02:14:05,225 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:05,225 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:05,225 INFO  [Listener,port=41411 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41411: stopping
2024-11-25T02:14:05,237 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=26 (was 19)
Potentially hanging thread: BRPC Connection (1107543386) to localhost:41411 from hbase/localhost@EXAMPLE.COM
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.hbase.ipc.BlockingRpcConnection.waitForWork(BlockingRpcConnection.java:329)
	app//org.apache.hadoop.hbase.ipc.BlockingRpcConnection.run(BlockingRpcConnection.java:342)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer idle connection scanner for port 41411
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:115)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63)

Potentially hanging thread: Listener,port=41411
	java.base@17.0.11/java.util.stream.Sink$ChainedReference.end(Sink.java:258)
	java.base@17.0.11/java.util.stream.Sink$ChainedReference.end(Sink.java:258)
	java.base@17.0.11/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:528)
	java.base@17.0.11/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513)
	java.base@17.0.11/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
	java.base@17.0.11/java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150)
	java.base@17.0.11/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
	java.base@17.0.11/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647)
	app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$15(StackLocator.java:100)
	app//org.apache.logging.log4j.util.StackLocator$$Lambda$125/0x00007f79e018c9a0.apply(Unknown Source)
	java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243)
	java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499)
	app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97)
	app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121)
	app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:252)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=2,bindAddress=localhost,port=41411
	java.base@17.0.11/java.lang.invoke.VarHandleBooleans$FieldInstanceReadWrite.compareAndSet(VarHandleBooleans.java:178)
	java.base@17.0.11/java.lang.invoke.VarHandleGuards.guard_LII_Z(VarHandleGuards.java:137)
	java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:125)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:151)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=0,bindAddress=localhost,port=41411
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=292 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 696), ProcessCount=11 (was 11), AvailableMemoryMB=4117 (was 4132)
2024-11-25T02:14:05,260 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=696, ProcessCount=11, AvailableMemoryMB=4116
2024-11-25T02:14:05,281 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:05,281 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:05,292 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500845291,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:05,296 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:05,312 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=36857
2024-11-25T02:14:05,316 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:05,317 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:05,317 INFO  [Listener,port=36857 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=36857: starting
2024-11-25T02:14:05,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12449adf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:05,319 DEBUG [Thread-41 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:05,320 DEBUG [Thread-41 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:36857
2024-11-25T02:14:05,329 DEBUG [Thread-41 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:05,338 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500845338,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:05,343 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:05,343 DEBUG [Reader=1,bindAddress=localhost,port=36857 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:05,345 DEBUG [Reader=1,bindAddress=localhost,port=36857 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:05,345 DEBUG [Reader=1,bindAddress=localhost,port=36857 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:05,347 DEBUG [Reader=1,bindAddress=localhost,port=36857 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:05,347 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:05,347 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:05,348 DEBUG [Reader=1,bindAddress=localhost,port=36857 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:05,348 DEBUG [Reader=1,bindAddress=localhost,port=36857 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:05,348 DEBUG [Reader=1,bindAddress=localhost,port=36857 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:05,352 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:05,353 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:05,353 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-25T02:14:05,353 DEBUG [Reader=1,bindAddress=localhost,port=36857 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:05,353 DEBUG [Thread-41 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:05,353 DEBUG [Reader=1,bindAddress=localhost,port=36857 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:05,353 DEBUG [Reader=1,bindAddress=localhost,port=36857 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:05,353 DEBUG [Reader=1,bindAddress=localhost,port=36857 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T02:14:05,353 INFO  [Reader=1,bindAddress=localhost,port=36857 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:05,354 INFO  [Reader=1,bindAddress=localhost,port=36857 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42700, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:05,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:05,374 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 36857
2024-11-25T02:14:05,374 INFO  [Listener,port=36857 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=36857: stopping
2024-11-25T02:14:05,375 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:05,376 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:05,376 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:05,392 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=35489
2024-11-25T02:14:05,404 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:05,412 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:05,412 INFO  [Listener,port=35489 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=35489: starting
2024-11-25T02:14:05,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b05003d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:05,417 DEBUG [Thread-44 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:05,417 DEBUG [Thread-44 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:35489
2024-11-25T02:14:05,418 DEBUG [Thread-44 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:05,421 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:05,421 DEBUG [Reader=1,bindAddress=localhost,port=35489 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:05,422 DEBUG [Reader=1,bindAddress=localhost,port=35489 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:05,422 DEBUG [Reader=1,bindAddress=localhost,port=35489 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:05,424 DEBUG [Reader=1,bindAddress=localhost,port=35489 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:05,424 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:05,425 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:05,426 DEBUG [Reader=1,bindAddress=localhost,port=35489 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:05,426 DEBUG [Reader=1,bindAddress=localhost,port=35489 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:05,426 DEBUG [Reader=1,bindAddress=localhost,port=35489 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:05,428 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:05,429 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:05,429 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-25T02:14:05,429 DEBUG [Thread-44 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:05,429 DEBUG [Reader=1,bindAddress=localhost,port=35489 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:05,429 DEBUG [Reader=1,bindAddress=localhost,port=35489 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:05,430 DEBUG [Reader=1,bindAddress=localhost,port=35489 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:05,430 DEBUG [Reader=1,bindAddress=localhost,port=35489 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T02:14:05,430 INFO  [Reader=1,bindAddress=localhost,port=35489 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:05,431 INFO  [Reader=1,bindAddress=localhost,port=35489 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56220, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:07,432 ERROR [Thread-44 {}] ipc.BlockingRpcConnection(646): Can't get the connection header response for rpc timeout, please check if server has the correct configuration to support the additional function.
java.net.SocketTimeoutException: 2000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:56220 remote=localhost/127.0.0.1:35489]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:163) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:132) ~[?:?]
	at java.io.DataInputStream.readFully(DataInputStream.java:201) ~[?:?]
	at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?]
	at org.apache.hadoop.security.SaslInputStream.readMoreData(SaslInputStream.java:97) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.security.SaslInputStream.read(SaslInputStream.java:260) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:263) ~[?:?]
	at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.processResponseForConnectionHeader(BlockingRpcConnection.java:630) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:570) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-25T02:14:07,433 DEBUG [Thread-44 {}] ipc.FailedServers(52): Added failed server with address localhost:35489 to list caused by java.io.IOException: Timeout while waiting connection header response
2024-11-25T02:14:07,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:07,434 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 35489
2024-11-25T02:14:07,434 INFO  [Listener,port=35489 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=35489: stopping
2024-11-25T02:14:07,435 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:07,435 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:07,442 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=21 (was 19)
Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150)
	java.base@17.0.11/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
	java.base@17.0.11/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647)
	app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$15(StackLocator.java:100)
	app//org.apache.logging.log4j.util.StackLocator$$Lambda$125/0x00007f79e018c9a0.apply(Unknown Source)
	java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243)
	java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499)
	app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97)
	app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121)
	app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63)

Potentially hanging thread: Reader=2,bindAddress=localhost,port=35489
	java.base@17.0.11/java.util.HashSet.iterator(HashSet.java:174)
	java.base@17.0.11/sun.nio.ch.Util$2.iterator(Util.java:353)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:169)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 696), ProcessCount=11 (was 11), AvailableMemoryMB=4059 (was 4116)
2024-11-25T02:14:07,449 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=696, ProcessCount=11, AvailableMemoryMB=4059
2024-11-25T02:14:07,462 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:07,462 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:07,467 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500847467,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:07,470 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:07,472 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=37591
2024-11-25T02:14:07,472 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:07,473 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:07,473 INFO  [Listener,port=37591 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=37591: starting
2024-11-25T02:14:07,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a7aa30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:07,474 DEBUG [Thread-47 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:07,474 DEBUG [Thread-47 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37591
2024-11-25T02:14:07,475 DEBUG [Thread-47 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:07,483 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500847482,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:07,485 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:07,485 DEBUG [Reader=1,bindAddress=localhost,port=37591 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:07,486 DEBUG [Reader=1,bindAddress=localhost,port=37591 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:07,487 DEBUG [Reader=1,bindAddress=localhost,port=37591 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:07,488 DEBUG [Reader=1,bindAddress=localhost,port=37591 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:07,488 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:07,489 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:07,489 DEBUG [Reader=1,bindAddress=localhost,port=37591 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:07,489 DEBUG [Reader=1,bindAddress=localhost,port=37591 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:07,489 DEBUG [Reader=1,bindAddress=localhost,port=37591 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:07,489 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:07,490 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:07,490 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T02:14:07,490 DEBUG [Thread-47 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:07,490 DEBUG [Reader=1,bindAddress=localhost,port=37591 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:07,490 DEBUG [Reader=1,bindAddress=localhost,port=37591 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:07,490 DEBUG [Reader=1,bindAddress=localhost,port=37591 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:07,490 DEBUG [Reader=1,bindAddress=localhost,port=37591 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:07,490 INFO  [Reader=1,bindAddress=localhost,port=37591 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:07,491 INFO  [Reader=1,bindAddress=localhost,port=37591 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58020, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:07,508 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:07,508 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 37591
2024-11-25T02:14:07,508 INFO  [Listener,port=37591 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=37591: stopping
2024-11-25T02:14:07,510 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:07,511 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:07,515 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=21 (was 19)
Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/sun.nio.ch.IOUtil.drain(Native Method)
	java.base@17.0.11/sun.nio.ch.EventFD.reset(EventFD.java:52)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.clearInterrupt(EPollSelectorImpl.java:263)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.processEvents(EPollSelectorImpl.java:201)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:135)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:115)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63)

Potentially hanging thread: Reader=2,bindAddress=localhost,port=37591
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=294 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 696), ProcessCount=11 (was 11), AvailableMemoryMB=4043 (was 4059)
2024-11-25T02:14:07,521 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=696, ProcessCount=11, AvailableMemoryMB=4043
2024-11-25T02:14:07,537 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:07,538 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:07,544 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500847544,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:07,548 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:07,550 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40935
2024-11-25T02:14:07,551 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:07,551 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:07,551 INFO  [Listener,port=40935 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40935: starting
2024-11-25T02:14:07,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3998ed0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:07,552 DEBUG [Thread-50 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:07,553 DEBUG [Thread-50 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40935
2024-11-25T02:14:07,554 DEBUG [Thread-50 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:07,560 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500847559,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:07,562 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:07,562 DEBUG [Reader=1,bindAddress=localhost,port=40935 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:07,563 DEBUG [Reader=1,bindAddress=localhost,port=40935 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:07,564 DEBUG [Reader=1,bindAddress=localhost,port=40935 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:07,566 DEBUG [Reader=1,bindAddress=localhost,port=40935 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:07,566 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:07,566 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:07,567 DEBUG [Reader=1,bindAddress=localhost,port=40935 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:07,567 DEBUG [Reader=1,bindAddress=localhost,port=40935 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:07,567 DEBUG [Reader=1,bindAddress=localhost,port=40935 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:07,567 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:07,567 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:07,567 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T02:14:07,568 DEBUG [Reader=1,bindAddress=localhost,port=40935 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:07,568 DEBUG [Thread-50 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:07,568 DEBUG [Reader=1,bindAddress=localhost,port=40935 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:07,568 DEBUG [Reader=1,bindAddress=localhost,port=40935 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:07,568 DEBUG [Reader=1,bindAddress=localhost,port=40935 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:07,568 INFO  [Reader=1,bindAddress=localhost,port=40935 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:07,568 INFO  [Reader=1,bindAddress=localhost,port=40935 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40356, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:07,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:07,594 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40935
2024-11-25T02:14:07,594 INFO  [Listener,port=40935 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40935: stopping
2024-11-25T02:14:07,595 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:07,595 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:07,612 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=23 (was 19)
Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=2,bindAddress=localhost,port=40935
	java.base@17.0.11/sun.nio.ch.IOUtil.drain(Native Method)
	java.base@17.0.11/sun.nio.ch.EventFD.reset(EventFD.java:52)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.clearInterrupt(EPollSelectorImpl.java:263)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.processEvents(EPollSelectorImpl.java:201)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:135)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=1,bindAddress=localhost,port=40935
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/java.lang.Thread.interrupt0(Native Method)
	java.base@17.0.11/java.lang.Thread.interrupt(Thread.java:1011)
	app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.write(EventChannelEncoder.java:306)
	app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.setOutErr(EventChannelEncoder.java:173)
	app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.testOutput(EventChannelEncoder.java:168)
	app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:99)
	app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:43)
	app//org.apache.maven.surefire.common.junit4.JUnit4RunListener.writeTestOutput(JUnit4RunListener.java:202)
	app//org.apache.maven.surefire.api.report.ConsoleOutputCapture$ForwardingPrintStream.write(ConsoleOutputCapture.java:57)
	app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:250)
	app//org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:283)
	app//org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:294)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:217)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199)
	app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158)
	app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89)
	app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552)
	app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
	app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
	app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 696), ProcessCount=11 (was 11), AvailableMemoryMB=4025 (was 4043)
2024-11-25T02:14:07,622 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=696, ProcessCount=11, AvailableMemoryMB=4023
2024-11-25T02:14:07,640 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:07,640 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:07,649 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500847648,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:07,652 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:07,657 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39537
2024-11-25T02:14:07,668 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:07,671 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:07,671 INFO  [Listener,port=39537 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39537: starting
2024-11-25T02:14:07,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15ec87e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:07,674 DEBUG [Thread-53 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:07,674 DEBUG [Thread-53 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39537
2024-11-25T02:14:07,676 DEBUG [Thread-53 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:07,684 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500847683,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:07,686 DEBUG [Thread-53 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:07,686 DEBUG [Thread-53 {}] security.HBaseSaslRpcClient(113): Server asks us to fall back to simple auth.
2024-11-25T02:14:07,686 DEBUG [Thread-53 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:07,687 INFO  [Reader=1,bindAddress=localhost,port=39537 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-25T02:14:07,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:07,713 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39537
2024-11-25T02:14:07,714 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:07,714 INFO  [Listener,port=39537 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39537: stopping
2024-11-25T02:14:07,714 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:07,720 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=24 (was 19)
Potentially hanging thread: RpcServer.responder
	app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeBytes(OutputStreamManager.java:353)
	app//org.apache.logging.log4j.core.layout.TextEncoderHelper.writeEncodedText(TextEncoderHelper.java:96)
	app//org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeText(TextEncoderHelper.java:65)
	app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:68)
	app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:32)
	app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:227)
	app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199)
	app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158)
	app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89)
	app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552)
	app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
	app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
	app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Listener,port=39537
	java.base@17.0.11/java.util.concurrent.ConcurrentHashMap.mappingCount(ConcurrentHashMap.java:2175)
	java.base@17.0.11/java.util.concurrent.ConcurrentHashMap$CollectionView.toArray(ConcurrentHashMap.java:4480)
	java.base@17.0.11/java.util.Collections$SetFromMap.toArray(Collections.java:5688)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$ConnectionManager.toArray(SimpleRpcServer.java:566)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$ConnectionManager.closeAll(SimpleRpcServer.java:623)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:266)

Potentially hanging thread: Reader=0,bindAddress=localhost,port=39537
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:170)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer idle connection scanner for port 39537
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 696), ProcessCount=11 (was 11), AvailableMemoryMB=4008 (was 4023)
2024-11-25T02:14:07,729 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=696, ProcessCount=11, AvailableMemoryMB=4007
2024-11-25T02:14:07,742 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:07,742 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:07,747 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500847746,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:07,750 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:07,765 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:07,770 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-25T02:14:07,861 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3627 (auto-detected)
2024-11-25T02:14:07,868 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected)
2024-11-25T02:14:07,900 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43439
2024-11-25T02:14:07,900 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:07,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ba13bf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:07,908 DEBUG [Thread-54 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:07,908 DEBUG [Thread-54 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43439
2024-11-25T02:14:07,912 DEBUG [Thread-54 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:07,920 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500847920,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:07,922 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:07,950 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:07,950 DEBUG [NettyRpcServer-2-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:07,952 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:07,957 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:07,957 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:07,958 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:07,959 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:07,959 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:07,960 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:07,960 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-25T02:14:07,960 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:07,960 DEBUG [Thread-54 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:07,960 DEBUG [NettyRpcServer-2-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:07,961 DEBUG [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T02:14:07,961 INFO  [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:07,969 DEBUG [Thread-54 {}] ipc.BlockingRpcConnection(634): Length of response for connection header:93
2024-11-25T02:14:07,972 INFO  [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40774, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:07,976 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:255
2024-11-25T02:14:07,977 DEBUG [BRPC Connection (1107543386) to localhost:43439 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-25T02:14:07,984 DEBUG [BRPC Connection (1107543386) to localhost:43439 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:241
2024-11-25T02:14:07,985 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:2274
2024-11-25T02:14:07,992 DEBUG [BRPC Connection (1107543386) to localhost:43439 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-25T02:14:07,992 DEBUG [BRPC Connection (1107543386) to localhost:43439 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:2260
2024-11-25T02:14:07,996 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:4117
2024-11-25T02:14:07,997 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:18167
2024-11-25T02:14:07,998 DEBUG [BRPC Connection (1107543386) to localhost:43439 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-25T02:14:08,010 DEBUG [BRPC Connection (1107543386) to localhost:43439 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:22270
2024-11-25T02:14:08,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:08,011 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43439
2024-11-25T02:14:08,025 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=22 (was 19)
Potentially hanging thread: NettyRpcServer-2-1
	jdk.unsupported@17.0.11/sun.misc.Unsafe.getInt(Unsafe.java:353)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.PlatformDependent0.getInt(PlatformDependent0.java:712)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.PlatformDependent.getInt(PlatformDependent.java:567)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventArray.getInt(EpollEventArray.java:115)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventArray.fd(EpollEventArray.java:109)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:465)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-2-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=299 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 696), ProcessCount=11 (was 11), AvailableMemoryMB=3979 (was 4007)
2024-11-25T02:14:08,033 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=21, OpenFileDescriptor=299, MaxFileDescriptor=1048576, SystemLoadAverage=696, ProcessCount=11, AvailableMemoryMB=3978
2024-11-25T02:14:08,047 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:08,047 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:08,052 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500848052,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:08,055 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:08,055 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:08,057 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45121
2024-11-25T02:14:08,058 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:08,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a349ca3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null
2024-11-25T02:14:08,059 DEBUG [Thread-55 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:08,060 DEBUG [Thread-55 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45121
2024-11-25T02:14:08,061 DEBUG [Thread-55 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,067 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500848067,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,069 DEBUG [Thread-55 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:08,069 DEBUG [Thread-55 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T02:14:08,069 DEBUG [Thread-55 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:45121
org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections.
	at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:111) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-25T02:14:08,070 WARN  [Thread-55 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732500848054
2024-11-25T02:14:08,074 DEBUG [Thread-55 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,077 DEBUG [Thread-55 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:08,078 DEBUG [Thread-55 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T02:14:08,078 DEBUG [Thread-55 {}] ipc.FailedServers(52): Added failed server with address localhost:45121 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,078 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:08,079 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45121
2024-11-25T02:14:08,086 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=24 (was 21)
Potentially hanging thread: NettyRpcServer-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=308 (was 299) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 696), ProcessCount=11 (was 11), AvailableMemoryMB=3963 (was 3978)
2024-11-25T02:14:08,096 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=24, OpenFileDescriptor=308, MaxFileDescriptor=1048576, SystemLoadAverage=696, ProcessCount=11, AvailableMemoryMB=3962
2024-11-25T02:14:08,111 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:08,111 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:08,117 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500848116,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:08,120 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:08,120 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:08,125 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37887
2024-11-25T02:14:08,125 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:08,126 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f055493, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:08,133 DEBUG [Thread-56 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:08,133 DEBUG [Thread-56 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37887
2024-11-25T02:14:08,134 DEBUG [Thread-56 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,146 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500848145,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,151 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:08,152 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,152 DEBUG [NettyRpcServer-4-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,162 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:08,163 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:08,163 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:08,164 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,164 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:08,165 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:08,168 DEBUG [Thread-56 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T02:14:08,169 DEBUG [Thread-56 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:37887
javax.security.sasl.SaslException: No common protection layer between client and server
	at com.sun.security.sasl.gsskerb.GssKrb5Client.doFinalHandshake(GssKrb5Client.java:270) ~[jdk.security.jgss:?]
	at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:203) ~[jdk.security.jgss:?]
	at org.apache.hadoop.hbase.security.AbstractHBaseSaslRpcClient.evaluateChallenge(AbstractHBaseSaslRpcClient.java:105) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:126) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-25T02:14:08,169 WARN  [Thread-56 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732500848118
2024-11-25T02:14:08,252 DEBUG [Thread-56 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,254 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:08,254 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,254 DEBUG [NettyRpcServer-4-3 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,256 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:08,256 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:08,256 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:08,257 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,257 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:08,257 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:08,257 DEBUG [Thread-56 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T02:14:08,258 DEBUG [Thread-56 {}] ipc.FailedServers(52): Added failed server with address localhost:37887 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,258 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:08,258 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37887
2024-11-25T02:14:08,265 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=27 (was 24)
Potentially hanging thread: NettyRpcServer-4-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-4-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-4-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=317 (was 308) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 696), ProcessCount=11 (was 11), AvailableMemoryMB=3945 (was 3962)
2024-11-25T02:14:08,271 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=27, OpenFileDescriptor=317, MaxFileDescriptor=1048576, SystemLoadAverage=696, ProcessCount=11, AvailableMemoryMB=3945
2024-11-25T02:14:08,284 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:08,285 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:08,292 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500848291,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:08,295 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:08,295 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:08,296 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33649
2024-11-25T02:14:08,297 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:08,297 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@345a112, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:08,305 DEBUG [Thread-57 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:08,305 DEBUG [Thread-57 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:33649
2024-11-25T02:14:08,306 DEBUG [Thread-57 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,312 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500848312,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,314 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:08,314 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,314 DEBUG [NettyRpcServer-5-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,316 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:08,317 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:08,317 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:08,318 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,318 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:08,319 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:08,319 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:08,319 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T02:14:08,319 DEBUG [Thread-57 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:08,319 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,319 DEBUG [NettyRpcServer-5-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,319 DEBUG [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:08,320 INFO  [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:08,320 INFO  [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50698, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:08,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:08,333 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33649
2024-11-25T02:14:08,334 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:08,334 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:08,335 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42429
2024-11-25T02:14:08,335 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:08,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8799f7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:08,336 DEBUG [Thread-58 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:08,337 DEBUG [Thread-58 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42429
2024-11-25T02:14:08,338 DEBUG [Thread-58 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,339 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:08,342 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,343 DEBUG [NettyRpcServer-6-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,346 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:08,347 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:08,347 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:08,348 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,348 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:08,349 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:08,349 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:08,350 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T02:14:08,350 DEBUG [Thread-58 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:08,350 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,350 DEBUG [NettyRpcServer-6-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,350 DEBUG [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:08,350 INFO  [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:08,351 INFO  [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50356, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:08,372 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:08,372 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42429
2024-11-25T02:14:08,373 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:08,374 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:08,377 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38115
2024-11-25T02:14:08,377 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:08,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29d0deab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:08,385 DEBUG [Thread-59 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:08,386 DEBUG [Thread-59 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:38115
2024-11-25T02:14:08,388 DEBUG [Thread-59 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,389 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:08,394 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,394 DEBUG [NettyRpcServer-7-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,396 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:08,397 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:08,397 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:08,398 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,398 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:08,399 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:08,400 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:08,400 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T02:14:08,400 DEBUG [Thread-59 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:08,401 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,402 DEBUG [NettyRpcServer-7-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,402 DEBUG [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:08,402 INFO  [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:08,403 INFO  [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37376, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:08,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:08,430 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38115
2024-11-25T02:14:08,436 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:08,441 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:08,447 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39049
2024-11-25T02:14:08,447 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:08,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ec8369e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:08,450 DEBUG [Thread-60 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:08,450 DEBUG [Thread-60 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39049
2024-11-25T02:14:08,457 DEBUG [Thread-60 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,460 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:08,460 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,460 DEBUG [NettyRpcServer-8-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,470 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:08,474 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:08,474 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:08,475 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,475 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:08,480 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:08,481 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:08,481 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-int
2024-11-25T02:14:08,481 DEBUG [Thread-60 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:08,481 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,482 DEBUG [NettyRpcServer-8-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,482 DEBUG [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int
2024-11-25T02:14:08,482 INFO  [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:08,483 INFO  [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49914, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:08,524 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:08,524 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39049
2024-11-25T02:14:08,529 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:08,529 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:08,530 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40153
2024-11-25T02:14:08,531 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:08,532 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f6da744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:08,541 DEBUG [Thread-61 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:08,541 DEBUG [Thread-61 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40153
2024-11-25T02:14:08,543 DEBUG [Thread-61 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,544 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:08,549 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,549 DEBUG [NettyRpcServer-9-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,552 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:08,553 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:08,553 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:08,555 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,555 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:08,557 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:08,557 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:08,557 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-25T02:14:08,557 DEBUG [Thread-61 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:08,558 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,558 DEBUG [NettyRpcServer-9-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,558 DEBUG [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T02:14:08,558 INFO  [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:08,559 INFO  [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47884, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:08,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:08,604 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40153
2024-11-25T02:14:08,620 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=39 (was 27)
Potentially hanging thread: NettyRpcServer-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: BRPC Connection (1107543386) to localhost:40153 from hbase/localhost@EXAMPLE.COM
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.hbase.ipc.BlockingRpcConnection.waitForWork(BlockingRpcConnection.java:329)
	app//org.apache.hadoop.hbase.ipc.BlockingRpcConnection.run(BlockingRpcConnection.java:342)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-9-2
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:638)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess(DefaultPromise.java:97)
	app//org.apache.hbase.thirdparty.io.netty.channel.group.DefaultChannelGroupFuture.setSuccess0(DefaultChannelGroupFuture.java:200)
	app//org.apache.hbase.thirdparty.io.netty.channel.group.DefaultChannelGroupFuture.access$400(DefaultChannelGroupFuture.java:41)
	app//org.apache.hbase.thirdparty.io.netty.channel.group.DefaultChannelGroupFuture$1.operationComplete(DefaultChannelGroupFuture.java:75)
	app//org.apache.hbase.thirdparty.io.netty.channel.group.DefaultChannelGroupFuture$1.operationComplete(DefaultChannelGroupFuture.java:48)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105)
	app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.safeSetSuccess(AbstractChannel.java:988)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:684)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:619)
	app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.close(DefaultChannelPipeline.java:1299)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeClose(AbstractChannelHandlerContext.java:755)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.access$1200(AbstractChannelHandlerContext.java:61)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext$11.run(AbstractChannelHandlerContext.java:738)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:405)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=362 (was 317) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=672 (was 696), ProcessCount=11 (was 11), AvailableMemoryMB=3934 (was 3945)
2024-11-25T02:14:08,630 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=37, OpenFileDescriptor=362, MaxFileDescriptor=1048576, SystemLoadAverage=672, ProcessCount=11, AvailableMemoryMB=3933
2024-11-25T02:14:08,650 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:08,651 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:08,657 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500848657,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:08,672 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:08,673 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:08,680 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43045
2024-11-25T02:14:08,681 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:08,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a8aad54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:08,686 DEBUG [Thread-62 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-25T02:14:08,687 DEBUG [Thread-62 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43045
2024-11-25T02:14:08,696 DEBUG [BRPC Connection (1107543386) to localhost:43045 from testuser {}] ipc.RpcConnection(424): process preamble call response with response type null
2024-11-25T02:14:08,696 WARN  [BRPC Connection (1107543386) to localhost:43045 from testuser {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call
2024-11-25T02:14:08,696 DEBUG [BRPC Connection (1107543386) to localhost:43045 from testuser {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes
2024-11-25T02:14:08,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:08,697 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43045
2024-11-25T02:14:08,705 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=39 (was 37)
Potentially hanging thread: NettyRpcServer-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=371 (was 362) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=672 (was 672), ProcessCount=11 (was 11), AvailableMemoryMB=3922 (was 3933)
2024-11-25T02:14:08,712 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=39, OpenFileDescriptor=371, MaxFileDescriptor=1048576, SystemLoadAverage=672, ProcessCount=11, AvailableMemoryMB=3922
2024-11-25T02:14:08,727 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:08,728 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:08,733 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500848733,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:08,736 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:08,737 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:08,741 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43223
2024-11-25T02:14:08,741 WARN  [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! *********
2024-11-25T02:14:08,741 WARN  [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients
2024-11-25T02:14:08,741 WARN  [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true).
2024-11-25T02:14:08,741 WARN  [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user
2024-11-25T02:14:08,741 WARN  [Time-limited test {}] ipc.RpcServer(380): impersonation is possible!
2024-11-25T02:14:08,741 WARN  [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible,
2024-11-25T02:14:08,741 WARN  [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml
2024-11-25T02:14:08,741 WARN  [Time-limited test {}] ipc.RpcServer(383): ****************************
2024-11-25T02:14:08,741 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:08,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@600532d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:08,753 DEBUG [Thread-63 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-25T02:14:08,753 DEBUG [Thread-63 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43223
2024-11-25T02:14:08,757 WARN  [NettyRpcServer-11-2 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1
2024-11-25T02:14:08,758 INFO  [NettyRpcServer-11-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-25T02:14:08,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:08,777 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43223
2024-11-25T02:14:08,790 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=41 (was 39)
Potentially hanging thread: NettyRpcServer-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-11-2
	app//org.apache.hbase.thirdparty.io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueue.poll(BaseMpscLinkedArrayQueue.java:294)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.shaded.org.jctools.queues.MpscUnboundedArrayQueue.poll(MpscUnboundedArrayQueue.java:23)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.pollTaskFrom(SingleThreadEventExecutor.java:215)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.pollTask(SingleThreadEventExecutor.java:210)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:485)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:405)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=380 (was 371) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=672 (was 672), ProcessCount=11 (was 11), AvailableMemoryMB=3910 (was 3922)
2024-11-25T02:14:08,798 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=41, OpenFileDescriptor=380, MaxFileDescriptor=1048576, SystemLoadAverage=672, ProcessCount=11, AvailableMemoryMB=3909
2024-11-25T02:14:08,815 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:08,815 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:08,820 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500848819,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:08,823 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:08,823 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:08,824 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42319
2024-11-25T02:14:08,824 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:08,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5818ca2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:08,829 DEBUG [Thread-64 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:08,829 DEBUG [Thread-64 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42319
2024-11-25T02:14:08,830 DEBUG [Thread-64 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,836 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500848836,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,838 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:08,838 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,839 DEBUG [NettyRpcServer-12-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,840 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:08,841 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:08,842 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:08,842 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,842 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:08,843 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:08,843 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:08,843 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T02:14:08,843 DEBUG [Thread-64 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:08,843 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,844 DEBUG [NettyRpcServer-12-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,844 DEBUG [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:08,844 INFO  [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:08,844 INFO  [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59650, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:08,853 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:08,853 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42319
2024-11-25T02:14:08,865 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=43 (was 41)
Potentially hanging thread: NettyRpcServer-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=389 (was 380) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=672 (was 672), ProcessCount=11 (was 11), AvailableMemoryMB=3892 (was 3909)
2024-11-25T02:14:08,878 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=43, OpenFileDescriptor=389, MaxFileDescriptor=1048576, SystemLoadAverage=672, ProcessCount=11, AvailableMemoryMB=3891
2024-11-25T02:14:08,898 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:08,899 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:08,906 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500848905,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:08,908 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:08,909 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:08,924 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37735
2024-11-25T02:14:08,924 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:08,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a5bc59e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:08,928 DEBUG [Thread-65 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:08,928 DEBUG [Thread-65 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37735
2024-11-25T02:14:08,930 DEBUG [Thread-65 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,944 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500848944,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,947 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:08,948 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,948 DEBUG [NettyRpcServer-13-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,957 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:08,963 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:08,963 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:08,964 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,964 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:08,967 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:08,968 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:08,968 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-25T02:14:08,968 DEBUG [Thread-65 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:08,971 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:08,972 DEBUG [NettyRpcServer-13-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:08,972 DEBUG [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T02:14:08,972 INFO  [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:08,982 INFO  [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58030, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:09,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:09,045 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37735
2024-11-25T02:14:09,046 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:09,046 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:09,054 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37651
2024-11-25T02:14:09,054 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:09,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c8362e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:09,061 DEBUG [Thread-66 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:09,061 DEBUG [Thread-66 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37651
2024-11-25T02:14:09,062 DEBUG [Thread-66 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:09,064 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:09,082 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:09,082 DEBUG [NettyRpcServer-14-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:09,084 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:09,085 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:09,086 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:09,086 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:09,086 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:09,088 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:09,089 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:09,089 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-25T02:14:09,089 DEBUG [Thread-66 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:09,089 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:09,089 DEBUG [NettyRpcServer-14-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:09,089 DEBUG [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T02:14:09,089 INFO  [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:09,091 INFO  [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53374, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:11,092 ERROR [Thread-66 {}] ipc.BlockingRpcConnection(646): Can't get the connection header response for rpc timeout, please check if server has the correct configuration to support the additional function.
java.net.SocketTimeoutException: 2000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:53374 remote=localhost/127.0.0.1:37651]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:163) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:132) ~[?:?]
	at java.io.DataInputStream.readFully(DataInputStream.java:201) ~[?:?]
	at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?]
	at org.apache.hadoop.security.SaslInputStream.readMoreData(SaslInputStream.java:97) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.security.SaslInputStream.read(SaslInputStream.java:260) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:263) ~[?:?]
	at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.processResponseForConnectionHeader(BlockingRpcConnection.java:630) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:570) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-25T02:14:11,092 DEBUG [Thread-66 {}] ipc.FailedServers(52): Added failed server with address localhost:37651 to list caused by java.io.IOException: Timeout while waiting connection header response
2024-11-25T02:14:11,092 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:11,092 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37651
2024-11-25T02:14:11,099 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=47 (was 43)
Potentially hanging thread: NettyRpcServer-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=407 (was 389) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=672 (was 672), ProcessCount=11 (was 11), AvailableMemoryMB=3863 (was 3891)
2024-11-25T02:14:11,105 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=47, OpenFileDescriptor=407, MaxFileDescriptor=1048576, SystemLoadAverage=672, ProcessCount=11, AvailableMemoryMB=3863
2024-11-25T02:14:11,117 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:11,117 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:11,122 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500851121,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:11,124 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:11,125 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:11,126 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36145
2024-11-25T02:14:11,126 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:11,126 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2663c5fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:11,127 DEBUG [Thread-67 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:11,127 DEBUG [Thread-67 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:36145
2024-11-25T02:14:11,128 DEBUG [Thread-67 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,135 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500851134,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,136 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:11,136 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,136 DEBUG [NettyRpcServer-15-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,138 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:11,139 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:11,139 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:11,140 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,140 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:11,141 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:11,141 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:11,141 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T02:14:11,141 DEBUG [Thread-67 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:11,141 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,142 DEBUG [NettyRpcServer-15-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,142 DEBUG [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:11,142 INFO  [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:11,142 INFO  [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54124, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:11,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:11,152 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36145
2024-11-25T02:14:11,158 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=49 (was 47)
Potentially hanging thread: NettyRpcServer-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=416 (was 407) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=672 (was 672), ProcessCount=11 (was 11), AvailableMemoryMB=3847 (was 3863)
2024-11-25T02:14:11,166 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=49, OpenFileDescriptor=416, MaxFileDescriptor=1048576, SystemLoadAverage=672, ProcessCount=11, AvailableMemoryMB=3846
2024-11-25T02:14:11,180 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:11,180 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:11,185 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500851184,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:11,187 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:11,188 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:11,189 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42791
2024-11-25T02:14:11,189 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:11,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f47b356, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:11,190 DEBUG [Thread-68 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:11,191 DEBUG [Thread-68 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42791
2024-11-25T02:14:11,192 DEBUG [Thread-68 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,197 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500851197,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,199 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:11,199 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,199 DEBUG [NettyRpcServer-16-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,201 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:11,201 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T02:14:11,202 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T02:14:11,202 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,202 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:11,203 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T02:14:11,203 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T02:14:11,203 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T02:14:11,203 DEBUG [Thread-68 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:11,203 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,203 DEBUG [NettyRpcServer-16-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,204 DEBUG [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:11,204 INFO  [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:11,204 INFO  [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47794, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:11,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:11,223 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42791
2024-11-25T02:14:11,232 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=51 (was 49)
Potentially hanging thread: NettyRpcServer-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=425 (was 416) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=672 (was 672), ProcessCount=11 (was 11), AvailableMemoryMB=3831 (was 3846)
2024-11-25T02:14:11,241 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=51, OpenFileDescriptor=425, MaxFileDescriptor=1048576, SystemLoadAverage=672, ProcessCount=11, AvailableMemoryMB=3831
2024-11-25T02:14:11,258 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:11,259 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:11,264 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500851263,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:11,267 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:11,267 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:11,269 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42061
2024-11-25T02:14:11,269 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:11,270 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@610bc292, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:11,272 DEBUG [Thread-69 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:11,273 DEBUG [Thread-69 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42061
2024-11-25T02:14:11,274 DEBUG [Thread-69 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,280 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500851280,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,282 DEBUG [Thread-69 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T02:14:11,282 DEBUG [Thread-69 {}] security.HBaseSaslRpcClient(113): Server asks us to fall back to simple auth.
2024-11-25T02:14:11,282 DEBUG [Thread-69 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:11,284 INFO  [NettyRpcServer-17-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-25T02:14:11,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:11,321 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42061
2024-11-25T02:14:11,332 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=55 (was 51)
Potentially hanging thread: BRPC Connection (1107543386) to localhost:42061 from hbase/localhost@EXAMPLE.COM
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.hbase.ipc.BlockingRpcConnection.waitForWork(BlockingRpcConnection.java:329)
	app//org.apache.hadoop.hbase.ipc.BlockingRpcConnection.run(BlockingRpcConnection.java:342)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=435 (was 425) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=672 (was 672), ProcessCount=11 (was 11), AvailableMemoryMB=3814 (was 3831)
2024-11-25T02:14:11,342 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=53, OpenFileDescriptor=434, MaxFileDescriptor=1048576, SystemLoadAverage=672, ProcessCount=11, AvailableMemoryMB=3814
2024-11-25T02:14:11,361 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:11,361 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:11,367 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500851367,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:11,370 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:11,376 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41639
2024-11-25T02:14:11,377 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:11,377 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:11,377 INFO  [Listener,port=41639 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41639: starting
2024-11-25T02:14:11,378 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1850bd6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:11,382 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-25T02:14:11,382 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-25T02:14:11,389 DEBUG [Thread-72 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:11,409 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,417 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500851416,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,420 DEBUG [Reader=1,bindAddress=localhost,port=41639 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,421 DEBUG [Reader=1,bindAddress=localhost,port=41639 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:11,421 DEBUG [Reader=1,bindAddress=localhost,port=41639 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,422 DEBUG [Reader=1,bindAddress=localhost,port=41639 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:11,424 DEBUG [Reader=1,bindAddress=localhost,port=41639 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:11,424 DEBUG [Reader=1,bindAddress=localhost,port=41639 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,424 DEBUG [Reader=1,bindAddress=localhost,port=41639 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:11,425 DEBUG [Reader=1,bindAddress=localhost,port=41639 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:11,425 DEBUG [Reader=1,bindAddress=localhost,port=41639 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,425 DEBUG [Reader=1,bindAddress=localhost,port=41639 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,425 DEBUG [Reader=1,bindAddress=localhost,port=41639 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T02:14:11,425 INFO  [Reader=1,bindAddress=localhost,port=41639 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:11,432 INFO  [Reader=1,bindAddress=localhost,port=41639 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42888, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:11,433 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:11,461 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:11,462 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41639
2024-11-25T02:14:11,462 INFO  [Listener,port=41639 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41639: stopping
2024-11-25T02:14:11,469 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:11,469 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:11,478 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=58 (was 53)
Potentially hanging thread: RpcServer idle connection scanner for port 41639
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/java.lang.Class.isArray(Native Method)
	java.base@17.0.11/java.lang.Class.getComponentType(Class.java:1227)
	java.base@17.0.11/jdk.internal.misc.Unsafe.checkPrimitiveArray(Unsafe.java:558)
	java.base@17.0.11/jdk.internal.misc.Unsafe.checkPrimitivePointer(Unsafe.java:579)
	java.base@17.0.11/jdk.internal.misc.Unsafe.copyMemoryChecks(Unsafe.java:832)
	java.base@17.0.11/jdk.internal.misc.Unsafe.copyMemory(Unsafe.java:800)
	java.base@17.0.11/jdk.internal.misc.ScopedMemoryAccess.copyMemoryInternal(ScopedMemoryAccess.java:167)
	java.base@17.0.11/jdk.internal.misc.ScopedMemoryAccess.copyMemory(ScopedMemoryAccess.java:149)
	java.base@17.0.11/java.nio.ByteBuffer.putBuffer(ByteBuffer.java:1102)
	java.base@17.0.11/java.nio.ByteBuffer.put(ByteBuffer.java:1009)
	java.base@17.0.11/java.nio.HeapByteBuffer.put(HeapByteBuffer.java:243)
	app//org.apache.logging.log4j.core.layout.ByteBufferDestinationHelper.writeToUnsynchronized(ByteBufferDestinationHelper.java:47)
	app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeBytes(OutputStreamManager.java:353)
	app//org.apache.logging.log4j.core.layout.TextEncoderHelper.writeEncodedText(TextEncoderHelper.java:96)
	app//org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeText(TextEncoderHelper.java:65)
	app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:68)
	app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:32)
	app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:227)
	app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199)
	app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158)
	app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89)
	app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552)
	app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
	app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
	app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1616)
	java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Listener,port=41639
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:255)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 434) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=672 (was 672), ProcessCount=11 (was 11), AvailableMemoryMB=3794 (was 3814)
2024-11-25T02:14:11,487 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=54, OpenFileDescriptor=468, MaxFileDescriptor=1048576, SystemLoadAverage=672, ProcessCount=11, AvailableMemoryMB=3794
2024-11-25T02:14:11,502 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:11,502 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:11,506 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500851506,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:11,509 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:11,518 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41857
2024-11-25T02:14:11,521 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:11,521 INFO  [Listener,port=41857 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41857: starting
2024-11-25T02:14:11,521 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:11,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75c7e858, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null
2024-11-25T02:14:11,523 DEBUG [Thread-75 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:11,525 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,531 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500851531,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,534 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T02:14:11,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:11,535 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.FailedServers(52): Added failed server with address localhost:41857 to list caused by org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections.
2024-11-25T02:14:11,535 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41857
2024-11-25T02:14:11,535 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:11,535 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:11,537 INFO  [Listener,port=41857 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41857: stopping
2024-11-25T02:14:11,544 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=60 (was 54)
Potentially hanging thread: RPCClient-NioEventLoopGroup-18-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer idle connection scanner for port 41857
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: Reader=2,bindAddress=localhost,port=41857
	java.base@17.0.11/java.util.concurrent.ConcurrentHashMap$BaseIterator.<init>(ConcurrentHashMap.java:3435)
	java.base@17.0.11/java.util.concurrent.ConcurrentHashMap$KeyIterator.<init>(ConcurrentHashMap.java:3454)
	java.base@17.0.11/java.util.concurrent.ConcurrentHashMap$KeySetView.iterator(ConcurrentHashMap.java:4635)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:188)
	java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:151)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Listener,port=41857
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:213)

Potentially hanging thread: Reader=1,bindAddress=localhost,port=41857
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=0,bindAddress=localhost,port=41857
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=469 (was 468) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=672 (was 672), ProcessCount=11 (was 11), AvailableMemoryMB=3778 (was 3794)
2024-11-25T02:14:11,551 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=55, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=672, ProcessCount=11, AvailableMemoryMB=3778
2024-11-25T02:14:11,567 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:11,568 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:11,573 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500851572,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:11,575 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:11,595 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40417
2024-11-25T02:14:11,596 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:11,596 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:11,596 INFO  [Listener,port=40417 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40417: starting
2024-11-25T02:14:11,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f77f1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:11,605 DEBUG [Thread-78 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:11,609 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,615 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500851615,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,617 DEBUG [Reader=1,bindAddress=localhost,port=40417 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,619 DEBUG [Reader=1,bindAddress=localhost,port=40417 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:11,619 DEBUG [Reader=1,bindAddress=localhost,port=40417 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,620 DEBUG [Reader=1,bindAddress=localhost,port=40417 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:11,622 DEBUG [Reader=1,bindAddress=localhost,port=40417 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:11,622 DEBUG [Reader=1,bindAddress=localhost,port=40417 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,622 DEBUG [Reader=1,bindAddress=localhost,port=40417 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:11,625 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T02:14:11,626 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.FailedServers(52): Added failed server with address localhost:40417 to list caused by javax.security.sasl.SaslException: No common protection layer between client and server
2024-11-25T02:14:11,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:11,626 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40417
2024-11-25T02:14:11,627 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:11,628 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:11,628 INFO  [Listener,port=40417 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40417: stopping
2024-11-25T02:14:11,636 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=62 (was 55)
Potentially hanging thread: Reader=2,bindAddress=localhost,port=40417
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Listener,port=40417
	java.base@17.0.11/sun.nio.ch.EPoll.ctl(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implDereg(EPollSelectorImpl.java:230)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:267)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:134)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:213)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer idle connection scanner for port 40417
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:523)
	java.base@17.0.11/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513)
	java.base@17.0.11/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
	java.base@17.0.11/java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150)
	java.base@17.0.11/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
	java.base@17.0.11/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647)
	app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$15(StackLocator.java:100)
	app//org.apache.logging.log4j.util.StackLocator$$Lambda$125/0x00007f79e018c9a0.apply(Unknown Source)
	java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243)
	java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499)
	app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97)
	app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121)
	app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65)

Potentially hanging thread: Relogin-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=0,bindAddress=localhost,port=40417
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=672 (was 672), ProcessCount=11 (was 11), AvailableMemoryMB=3764 (was 3778)
2024-11-25T02:14:11,645 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=57, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=672, ProcessCount=11, AvailableMemoryMB=3763
2024-11-25T02:14:11,654 WARN  [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732500851574
2024-11-25T02:14:11,659 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:11,659 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:11,692 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500851691,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:11,694 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:11,697 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39047
2024-11-25T02:14:11,700 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:11,700 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:11,700 INFO  [Listener,port=39047 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39047: starting
2024-11-25T02:14:11,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b8d6c28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:11,701 DEBUG [Thread-81 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:11,703 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,706 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500851706,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,708 DEBUG [Reader=1,bindAddress=localhost,port=39047 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,708 DEBUG [Reader=1,bindAddress=localhost,port=39047 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:11,708 DEBUG [Reader=1,bindAddress=localhost,port=39047 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,709 DEBUG [Reader=1,bindAddress=localhost,port=39047 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:11,710 DEBUG [Reader=1,bindAddress=localhost,port=39047 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:11,710 DEBUG [Reader=1,bindAddress=localhost,port=39047 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,710 DEBUG [Reader=1,bindAddress=localhost,port=39047 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:11,711 DEBUG [Reader=1,bindAddress=localhost,port=39047 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:11,711 DEBUG [Reader=1,bindAddress=localhost,port=39047 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,711 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:11,711 DEBUG [Reader=1,bindAddress=localhost,port=39047 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,711 DEBUG [Reader=1,bindAddress=localhost,port=39047 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:11,711 INFO  [Reader=1,bindAddress=localhost,port=39047 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:11,712 INFO  [Reader=1,bindAddress=localhost,port=39047 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41572, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:11,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:11,719 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39047
2024-11-25T02:14:11,720 INFO  [Listener,port=39047 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39047: stopping
2024-11-25T02:14:11,720 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:11,720 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:11,720 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:11,728 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=36885
2024-11-25T02:14:11,728 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:11,729 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:11,729 INFO  [Listener,port=36885 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=36885: starting
2024-11-25T02:14:11,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41bb475, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:11,733 DEBUG [Thread-84 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:11,735 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,737 DEBUG [Reader=1,bindAddress=localhost,port=36885 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,738 DEBUG [Reader=1,bindAddress=localhost,port=36885 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:11,738 DEBUG [Reader=1,bindAddress=localhost,port=36885 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,739 DEBUG [Reader=1,bindAddress=localhost,port=36885 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:11,740 DEBUG [Reader=1,bindAddress=localhost,port=36885 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:11,740 DEBUG [Reader=1,bindAddress=localhost,port=36885 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,741 DEBUG [Reader=1,bindAddress=localhost,port=36885 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:11,741 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:11,742 DEBUG [Reader=1,bindAddress=localhost,port=36885 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:11,742 DEBUG [Reader=1,bindAddress=localhost,port=36885 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,742 DEBUG [Reader=1,bindAddress=localhost,port=36885 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,742 DEBUG [Reader=1,bindAddress=localhost,port=36885 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:11,742 INFO  [Reader=1,bindAddress=localhost,port=36885 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:11,743 INFO  [Reader=1,bindAddress=localhost,port=36885 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51428, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:11,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:11,751 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 36885
2024-11-25T02:14:11,751 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:11,751 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:11,752 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:11,752 INFO  [Listener,port=36885 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=36885: stopping
2024-11-25T02:14:11,758 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40199
2024-11-25T02:14:11,763 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:11,764 INFO  [Listener,port=40199 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40199: starting
2024-11-25T02:14:11,763 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:11,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cfe4583, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:11,773 DEBUG [Thread-87 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:11,777 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,779 DEBUG [Reader=1,bindAddress=localhost,port=40199 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,780 DEBUG [Reader=1,bindAddress=localhost,port=40199 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:11,780 DEBUG [Reader=1,bindAddress=localhost,port=40199 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,782 DEBUG [Reader=1,bindAddress=localhost,port=40199 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:11,783 DEBUG [Reader=1,bindAddress=localhost,port=40199 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:11,783 DEBUG [Reader=1,bindAddress=localhost,port=40199 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,783 DEBUG [Reader=1,bindAddress=localhost,port=40199 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:11,784 DEBUG [Reader=1,bindAddress=localhost,port=40199 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:11,784 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:11,784 DEBUG [Reader=1,bindAddress=localhost,port=40199 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,784 DEBUG [Reader=1,bindAddress=localhost,port=40199 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,784 DEBUG [Reader=1,bindAddress=localhost,port=40199 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:11,784 INFO  [Reader=1,bindAddress=localhost,port=40199 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:11,785 INFO  [Reader=1,bindAddress=localhost,port=40199 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53860, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:11,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:11,792 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40199
2024-11-25T02:14:11,792 INFO  [Listener,port=40199 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40199: stopping
2024-11-25T02:14:11,793 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:11,794 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:11,794 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:11,795 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41507
2024-11-25T02:14:11,801 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:11,801 INFO  [Listener,port=41507 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41507: starting
2024-11-25T02:14:11,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12d19706, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:11,802 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:11,802 DEBUG [Thread-90 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:11,804 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,806 DEBUG [Reader=1,bindAddress=localhost,port=41507 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,807 DEBUG [Reader=1,bindAddress=localhost,port=41507 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:11,807 DEBUG [Reader=1,bindAddress=localhost,port=41507 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,808 DEBUG [Reader=1,bindAddress=localhost,port=41507 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:11,811 DEBUG [Reader=1,bindAddress=localhost,port=41507 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:11,811 DEBUG [Reader=1,bindAddress=localhost,port=41507 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,812 DEBUG [Reader=1,bindAddress=localhost,port=41507 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:11,813 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:11,814 DEBUG [Reader=1,bindAddress=localhost,port=41507 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:11,814 DEBUG [Reader=1,bindAddress=localhost,port=41507 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,814 DEBUG [Reader=1,bindAddress=localhost,port=41507 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,814 DEBUG [Reader=1,bindAddress=localhost,port=41507 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int
2024-11-25T02:14:11,815 INFO  [Reader=1,bindAddress=localhost,port=41507 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:11,815 INFO  [Reader=1,bindAddress=localhost,port=41507 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56352, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:11,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:11,825 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41507
2024-11-25T02:14:11,825 INFO  [Listener,port=41507 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41507: stopping
2024-11-25T02:14:11,825 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:11,825 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:11,826 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:11,827 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34253
2024-11-25T02:14:11,827 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:11,827 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:11,827 INFO  [Listener,port=34253 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34253: starting
2024-11-25T02:14:11,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5134ade5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:11,828 DEBUG [Thread-93 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:11,830 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,832 DEBUG [Reader=1,bindAddress=localhost,port=34253 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,833 DEBUG [Reader=1,bindAddress=localhost,port=34253 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:11,833 DEBUG [Reader=1,bindAddress=localhost,port=34253 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,834 DEBUG [Reader=1,bindAddress=localhost,port=34253 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:11,835 DEBUG [Reader=1,bindAddress=localhost,port=34253 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:11,835 DEBUG [Reader=1,bindAddress=localhost,port=34253 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,835 DEBUG [Reader=1,bindAddress=localhost,port=34253 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:11,835 DEBUG [Reader=1,bindAddress=localhost,port=34253 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:11,835 DEBUG [Reader=1,bindAddress=localhost,port=34253 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:11,835 DEBUG [Reader=1,bindAddress=localhost,port=34253 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:11,836 DEBUG [Reader=1,bindAddress=localhost,port=34253 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T02:14:11,836 INFO  [Reader=1,bindAddress=localhost,port=34253 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:11,836 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:11,837 INFO  [Reader=1,bindAddress=localhost,port=34253 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48904, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:11,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:11,846 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34253
2024-11-25T02:14:11,846 INFO  [Listener,port=34253 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34253: stopping
2024-11-25T02:14:11,846 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:11,846 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:11,855 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=64 (was 57)
Potentially hanging thread: RPCClient-NioEventLoopGroup-18-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer idle connection scanner for port 34253
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=2,bindAddress=localhost,port=34253
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=672 (was 672), ProcessCount=11 (was 11), AvailableMemoryMB=3737 (was 3763)
2024-11-25T02:14:11,862 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=62, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=672, ProcessCount=11, AvailableMemoryMB=3737
2024-11-25T02:14:11,873 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:11,873 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:11,875 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500851875,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:11,877 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:11,880 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45803
2024-11-25T02:14:11,882 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:11,882 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:11,882 INFO  [Listener,port=45803 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45803: starting
2024-11-25T02:14:11,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554d60b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:11,883 DEBUG [Thread-96 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-25T02:14:11,889 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(424): process preamble call response with response type null
2024-11-25T02:14:11,889 WARN  [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call
2024-11-25T02:14:11,890 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes
2024-11-25T02:14:11,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:11,891 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45803
2024-11-25T02:14:11,891 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:11,891 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:11,892 INFO  [Listener,port=45803 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45803: stopping
2024-11-25T02:14:11,898 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=65 (was 62)
Potentially hanging thread: RpcServer idle connection scanner for port 45803
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: Listener,port=45803
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:213)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=672 (was 672), ProcessCount=11 (was 11), AvailableMemoryMB=3736 (was 3737)
2024-11-25T02:14:11,905 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=63, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=672, ProcessCount=11, AvailableMemoryMB=3736
2024-11-25T02:14:11,919 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:11,920 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:11,923 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500851922,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:11,930 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:11,936 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39493
2024-11-25T02:14:11,944 WARN  [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! *********
2024-11-25T02:14:11,944 WARN  [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients
2024-11-25T02:14:11,944 WARN  [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true).
2024-11-25T02:14:11,944 WARN  [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user
2024-11-25T02:14:11,944 WARN  [Time-limited test {}] ipc.RpcServer(380): impersonation is possible!
2024-11-25T02:14:11,944 WARN  [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible,
2024-11-25T02:14:11,944 WARN  [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml
2024-11-25T02:14:11,944 WARN  [Time-limited test {}] ipc.RpcServer(383): ****************************
2024-11-25T02:14:11,947 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:11,948 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:11,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@218952ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:11,951 INFO  [Listener,port=39493 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39493: starting
2024-11-25T02:14:11,951 DEBUG [Thread-99 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-25T02:14:11,955 WARN  [Reader=1,bindAddress=localhost,port=39493 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1
2024-11-25T02:14:11,955 INFO  [Reader=1,bindAddress=localhost,port=39493 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-25T02:14:11,964 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:11,964 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39493
2024-11-25T02:14:11,964 INFO  [Listener,port=39493 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39493: stopping
2024-11-25T02:14:11,970 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:11,970 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:11,975 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=65 (was 63) - Thread LEAK? -, OpenFileDescriptor=468 (was 466) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=672 (was 672), ProcessCount=11 (was 11), AvailableMemoryMB=3736 (was 3736)
2024-11-25T02:14:11,981 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=64, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=672, ProcessCount=11, AvailableMemoryMB=3736
2024-11-25T02:14:11,993 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:11,993 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:11,995 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500851995,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:11,998 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:12,000 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40679
2024-11-25T02:14:12,003 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:12,003 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:12,003 INFO  [Listener,port=40679 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40679: starting
2024-11-25T02:14:12,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bcd14cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:12,004 DEBUG [Thread-102 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:12,008 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:12,012 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500852011,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:12,013 DEBUG [Reader=1,bindAddress=localhost,port=40679 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:12,014 DEBUG [Reader=1,bindAddress=localhost,port=40679 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:12,014 DEBUG [Reader=1,bindAddress=localhost,port=40679 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=498 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:12,015 DEBUG [Reader=1,bindAddress=localhost,port=40679 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 107 from saslServer.
2024-11-25T02:14:12,016 DEBUG [Reader=1,bindAddress=localhost,port=40679 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:12,016 DEBUG [Reader=1,bindAddress=localhost,port=40679 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:12,016 DEBUG [Reader=1,bindAddress=localhost,port=40679 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:12,018 DEBUG [Reader=1,bindAddress=localhost,port=40679 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:12,018 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:12,018 DEBUG [Reader=1,bindAddress=localhost,port=40679 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:12,019 DEBUG [Reader=1,bindAddress=localhost,port=40679 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:12,019 DEBUG [Reader=1,bindAddress=localhost,port=40679 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:12,019 INFO  [Reader=1,bindAddress=localhost,port=40679 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:12,019 INFO  [Reader=1,bindAddress=localhost,port=40679 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47460, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:12,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:12,031 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40679
2024-11-25T02:14:12,031 INFO  [Listener,port=40679 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40679: stopping
2024-11-25T02:14:12,031 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:12,031 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:12,045 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=71 (was 64)
Potentially hanging thread: RPCClient-NioEventLoopGroup-18-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=2,bindAddress=localhost,port=40679
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=0,bindAddress=localhost,port=40679
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer.responder
	app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
	app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
	app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65)

Potentially hanging thread: RpcServer idle connection scanner for port 40679
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: Reader=1,bindAddress=localhost,port=40679
	java.base@17.0.11/sun.nio.ch.FileDispatcherImpl.closeIntFD(Native Method)
	java.base@17.0.11/sun.nio.ch.EventFD.close(EventFD.java:56)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implClose(EPollSelectorImpl.java:219)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:185)
	java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:151)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=672 (was 672), ProcessCount=11 (was 11), AvailableMemoryMB=3732 (was 3736)
2024-11-25T02:14:12,052 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=65, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=672, ProcessCount=11, AvailableMemoryMB=3732
2024-11-25T02:14:12,064 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:12,064 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:12,067 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500852067,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:12,069 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:12,071 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40817
2024-11-25T02:14:12,071 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:12,071 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:12,071 INFO  [Listener,port=40817 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40817: starting
2024-11-25T02:14:12,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@552333d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:12,072 DEBUG [Thread-105 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:12,078 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:12,081 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500852081,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:12,084 DEBUG [Reader=1,bindAddress=localhost,port=40817 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:12,085 DEBUG [Reader=1,bindAddress=localhost,port=40817 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:12,085 DEBUG [Reader=1,bindAddress=localhost,port=40817 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:12,087 DEBUG [Reader=1,bindAddress=localhost,port=40817 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:12,089 DEBUG [Reader=1,bindAddress=localhost,port=40817 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:12,089 DEBUG [Reader=1,bindAddress=localhost,port=40817 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:12,089 DEBUG [Reader=1,bindAddress=localhost,port=40817 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:12,092 DEBUG [Reader=1,bindAddress=localhost,port=40817 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:12,093 DEBUG [Reader=1,bindAddress=localhost,port=40817 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:12,093 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:12,093 DEBUG [Reader=1,bindAddress=localhost,port=40817 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:12,093 DEBUG [Reader=1,bindAddress=localhost,port=40817 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T02:14:12,093 INFO  [Reader=1,bindAddress=localhost,port=40817 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:12,094 INFO  [Reader=1,bindAddress=localhost,port=40817 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54528, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:12,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:12,129 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40817
2024-11-25T02:14:12,130 INFO  [Listener,port=40817 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40817: stopping
2024-11-25T02:14:12,130 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:12,130 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:12,131 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:12,136 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44729
2024-11-25T02:14:12,141 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:12,143 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:12,144 INFO  [Listener,port=44729 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44729: starting
2024-11-25T02:14:12,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43de16ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:12,147 DEBUG [Thread-108 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:12,150 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:12,151 DEBUG [Reader=1,bindAddress=localhost,port=44729 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:12,152 DEBUG [Reader=1,bindAddress=localhost,port=44729 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:12,153 DEBUG [Reader=1,bindAddress=localhost,port=44729 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:12,154 DEBUG [Reader=1,bindAddress=localhost,port=44729 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:12,158 DEBUG [Reader=1,bindAddress=localhost,port=44729 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:12,158 DEBUG [Reader=1,bindAddress=localhost,port=44729 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:12,158 DEBUG [Reader=1,bindAddress=localhost,port=44729 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:12,160 DEBUG [Reader=1,bindAddress=localhost,port=44729 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:12,160 DEBUG [Reader=1,bindAddress=localhost,port=44729 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:12,160 DEBUG [Reader=1,bindAddress=localhost,port=44729 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:12,161 DEBUG [Reader=1,bindAddress=localhost,port=44729 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T02:14:12,161 INFO  [Reader=1,bindAddress=localhost,port=44729 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:12,161 INFO  [Reader=1,bindAddress=localhost,port=44729 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50994, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:14,161 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T02:14:14,161 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:14,162 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.FailedServers(52): Added failed server with address localhost:44729 to list caused by java.io.IOException: org.apache.hbase.thirdparty.io.netty.handler.timeout.ReadTimeoutException
2024-11-25T02:14:14,162 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44729
2024-11-25T02:14:14,162 INFO  [Listener,port=44729 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44729: stopping
2024-11-25T02:14:14,163 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:14,163 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:14,170 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=70 (was 65)
Potentially hanging thread: RPCClient-NioEventLoopGroup-18-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=2,bindAddress=localhost,port=44729
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=0,bindAddress=localhost,port=44729
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:169)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:115)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=738 (was 672) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3717 (was 3732)
2024-11-25T02:14:14,172 WARN  [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732500852068
2024-11-25T02:14:14,176 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=67, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=738, ProcessCount=11, AvailableMemoryMB=3717
2024-11-25T02:14:14,188 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:14,188 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:14,190 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500854190,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:14,193 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:14,194 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=46621
2024-11-25T02:14:14,194 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:14,194 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:14,194 INFO  [Listener,port=46621 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=46621: starting
2024-11-25T02:14:14,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5460e532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:14,195 DEBUG [Thread-111 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:14,197 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,200 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500854199,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,201 DEBUG [Reader=1,bindAddress=localhost,port=46621 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,202 DEBUG [Reader=1,bindAddress=localhost,port=46621 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:14,202 DEBUG [Reader=1,bindAddress=localhost,port=46621 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,203 DEBUG [Reader=1,bindAddress=localhost,port=46621 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:14,204 DEBUG [Reader=1,bindAddress=localhost,port=46621 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:14,204 DEBUG [Reader=1,bindAddress=localhost,port=46621 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,204 DEBUG [Reader=1,bindAddress=localhost,port=46621 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:14,205 DEBUG [Reader=1,bindAddress=localhost,port=46621 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:14,205 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:14,205 DEBUG [Reader=1,bindAddress=localhost,port=46621 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,205 DEBUG [Reader=1,bindAddress=localhost,port=46621 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,205 DEBUG [Reader=1,bindAddress=localhost,port=46621 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:14,205 INFO  [Reader=1,bindAddress=localhost,port=46621 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:14,205 INFO  [Reader=1,bindAddress=localhost,port=46621 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37346, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:14,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:14,212 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 46621
2024-11-25T02:14:14,212 INFO  [Listener,port=46621 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=46621: stopping
2024-11-25T02:14:14,212 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:14,212 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:14,220 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=69 (was 67) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=738 (was 738), ProcessCount=11 (was 11), AvailableMemoryMB=3717 (was 3717)
2024-11-25T02:14:14,227 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=68, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=738, ProcessCount=11, AvailableMemoryMB=3716
2024-11-25T02:14:14,238 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:14,238 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:14,241 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500854240,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:14,243 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:14,244 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44411
2024-11-25T02:14:14,245 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:14,248 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:14,248 INFO  [Listener,port=44411 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44411: starting
2024-11-25T02:14:14,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@628fdd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:14,256 DEBUG [Thread-114 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:14,258 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,261 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500854261,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,263 DEBUG [Reader=1,bindAddress=localhost,port=44411 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,264 DEBUG [Reader=1,bindAddress=localhost,port=44411 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:14,264 DEBUG [Reader=1,bindAddress=localhost,port=44411 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,265 DEBUG [Reader=1,bindAddress=localhost,port=44411 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T02:14:14,265 DEBUG [Reader=1,bindAddress=localhost,port=44411 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:14,266 DEBUG [Reader=1,bindAddress=localhost,port=44411 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,266 DEBUG [Reader=1,bindAddress=localhost,port=44411 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T02:14:14,266 DEBUG [Reader=1,bindAddress=localhost,port=44411 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T02:14:14,266 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:14,266 DEBUG [Reader=1,bindAddress=localhost,port=44411 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,266 DEBUG [Reader=1,bindAddress=localhost,port=44411 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,266 DEBUG [Reader=1,bindAddress=localhost,port=44411 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:14,267 INFO  [Reader=1,bindAddress=localhost,port=44411 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:14,267 INFO  [Reader=1,bindAddress=localhost,port=44411 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33192, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:14,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:14,275 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44411
2024-11-25T02:14:14,275 INFO  [Listener,port=44411 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44411: stopping
2024-11-25T02:14:14,275 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:14,275 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:14,288 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=75 (was 68)
Potentially hanging thread: RpcServer idle connection scanner for port 44411
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=0,bindAddress=localhost,port=44411
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/sun.nio.ch.FileDispatcherImpl.closeIntFD(Native Method)
	java.base@17.0.11/sun.nio.ch.EventFD.close(EventFD.java:56)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implClose(EPollSelectorImpl.java:219)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:185)
	java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:67)

Potentially hanging thread: Listener,port=44411
	java.base@17.0.11/sun.nio.ch.FileDispatcherImpl.closeIntFD(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implClose(EPollSelectorImpl.java:216)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:185)
	java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:256)

Potentially hanging thread: Reader=1,bindAddress=localhost,port=44411
	java.base@17.0.11/java.lang.ref.Reference.refersTo0(Native Method)
	java.base@17.0.11/java.lang.ref.Reference.refersToImpl(Reference.java:375)
	java.base@17.0.11/java.lang.ref.Reference.refersTo(Reference.java:366)
	java.base@17.0.11/java.lang.ThreadLocal$ThreadLocalMap.getEntry(ThreadLocal.java:436)
	java.base@17.0.11/java.lang.ThreadLocal.isPresent(ThreadLocal.java:185)
	java.base@17.0.11/java.lang.Thread.exit(Thread.java:849)
 - Thread LEAK? -, OpenFileDescriptor=468 (was 466) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=738 (was 738), ProcessCount=11 (was 11), AvailableMemoryMB=3716 (was 3716)
2024-11-25T02:14:14,298 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=69, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=738, ProcessCount=11, AvailableMemoryMB=3716
2024-11-25T02:14:14,312 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:14,312 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:14,314 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500854314,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:14,317 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:14,320 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=33711
2024-11-25T02:14:14,321 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T02:14:14,321 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:14,322 INFO  [Listener,port=33711 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=33711: starting
2024-11-25T02:14:14,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c786677, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:14,323 DEBUG [Thread-117 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:14,330 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,335 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500854335,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,340 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:14,341 INFO  [Reader=1,bindAddress=localhost,port=33711 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-25T02:14:14,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:14,352 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 33711
2024-11-25T02:14:14,352 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T02:14:14,352 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T02:14:14,356 INFO  [Listener,port=33711 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=33711: stopping
2024-11-25T02:14:14,360 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=74 (was 69)
Potentially hanging thread: RpcServer idle connection scanner for port 33711
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Listener,port=33711
	java.base@17.0.11/java.lang.StackTraceElement.initStackTraceElement(Native Method)
	java.base@17.0.11/java.lang.StackTraceElement.of(StackTraceElement.java:555)
	java.base@17.0.11/java.lang.StackFrameInfo.toStackTraceElement(StackFrameInfo.java:130)
	app//org.apache.logging.log4j.util.StackLocator$$Lambda$128/0x00007f79e018d098.apply(Unknown Source)
	java.base@17.0.11/java.util.Optional.map(Optional.java:260)
	app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:100)
	app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121)
	app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:252)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=1,bindAddress=localhost,port=33711
	java.base@17.0.11/sun.nio.ch.Util$1.threadTerminated(Util.java:63)
	java.base@17.0.11/sun.nio.ch.Util$1.threadTerminated(Util.java:56)
	java.base@17.0.11/jdk.internal.misc.TerminatingThreadLocal._threadTerminated(TerminatingThreadLocal.java:74)
	java.base@17.0.11/jdk.internal.misc.TerminatingThreadLocal.threadTerminated(TerminatingThreadLocal.java:70)
	java.base@17.0.11/java.lang.Thread.exit(Thread.java:850)
 - Thread LEAK? -, OpenFileDescriptor=469 (was 466) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=738 (was 738), ProcessCount=11 (was 11), AvailableMemoryMB=3715 (was 3716)
2024-11-25T02:14:14,368 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=70, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=738, ProcessCount=11, AvailableMemoryMB=3715
2024-11-25T02:14:14,381 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:14,382 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:14,384 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500854384,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:14,386 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:14,387 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:14,388 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43991
2024-11-25T02:14:14,388 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:14,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e770dcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:14,391 DEBUG [Thread-118 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:14,392 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,398 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500854398,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,400 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,400 DEBUG [NettyRpcServer-19-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,402 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:14,403 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,403 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:14,404 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,404 DEBUG [NettyRpcServer-19-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,404 DEBUG [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T02:14:14,404 INFO  [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:14,406 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:14,407 INFO  [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36850, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:14,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:14,436 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43991
2024-11-25T02:14:14,445 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=73 (was 70)
Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-19-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-19-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=475 (was 466) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=738 (was 738), ProcessCount=11 (was 11), AvailableMemoryMB=3714 (was 3715)
2024-11-25T02:14:14,456 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=72, OpenFileDescriptor=475, MaxFileDescriptor=1048576, SystemLoadAverage=738, ProcessCount=11, AvailableMemoryMB=3713
2024-11-25T02:14:14,469 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:14,469 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:14,472 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500854472,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:14,477 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:14,478 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:14,482 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45883
2024-11-25T02:14:14,482 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:14,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6360ce81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null
2024-11-25T02:14:14,488 DEBUG [Thread-119 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:14,494 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,504 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500854503,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,508 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T02:14:14,509 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.FailedServers(52): Added failed server with address localhost:45883 to list caused by org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections.
2024-11-25T02:14:14,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:14,511 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45883
2024-11-25T02:14:14,526 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=74 (was 72) - Thread LEAK? -, OpenFileDescriptor=484 (was 475) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=738 (was 738), ProcessCount=11 (was 11), AvailableMemoryMB=3709 (was 3713)
2024-11-25T02:14:14,541 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=74, OpenFileDescriptor=484, MaxFileDescriptor=1048576, SystemLoadAverage=738, ProcessCount=11, AvailableMemoryMB=3709
2024-11-25T02:14:14,557 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:14,557 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:14,561 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500854561,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:14,564 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:14,564 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:14,568 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46815
2024-11-25T02:14:14,568 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:14,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51b9d9e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:14,575 DEBUG [Thread-120 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:14,577 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,580 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500854580,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,587 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,587 DEBUG [NettyRpcServer-21-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,589 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:14,596 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,596 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:14,598 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T02:14:14,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:14,598 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.FailedServers(52): Added failed server with address localhost:46815 to list caused by javax.security.sasl.SaslException: No common protection layer between client and server
2024-11-25T02:14:14,598 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46815
2024-11-25T02:14:14,610 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=76 (was 74) - Thread LEAK? -, OpenFileDescriptor=493 (was 484) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=738 (was 738), ProcessCount=11 (was 11), AvailableMemoryMB=3703 (was 3709)
2024-11-25T02:14:14,621 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=76, OpenFileDescriptor=493, MaxFileDescriptor=1048576, SystemLoadAverage=738, ProcessCount=11, AvailableMemoryMB=3702
2024-11-25T02:14:14,639 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:14,639 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:14,642 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500854641,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:14,644 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:14,644 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:14,648 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39359
2024-11-25T02:14:14,649 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:14,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@638a668b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:14,653 DEBUG [Thread-121 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:14,657 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,661 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500854661,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,668 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,668 DEBUG [NettyRpcServer-22-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,670 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:14,671 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,671 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:14,672 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:14,673 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,673 DEBUG [NettyRpcServer-22-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,673 DEBUG [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:14,673 INFO  [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:14,674 INFO  [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49582, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:14,687 WARN  [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732500854643
2024-11-25T02:14:14,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:14,698 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39359
2024-11-25T02:14:14,699 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:14,699 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:14,733 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44223
2024-11-25T02:14:14,733 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:14,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48504c57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:14,734 DEBUG [Thread-122 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:14,737 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,741 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,741 DEBUG [NettyRpcServer-23-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,742 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:14,743 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,744 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:14,744 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:14,744 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,744 DEBUG [NettyRpcServer-23-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,745 DEBUG [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:14,745 INFO  [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:14,745 INFO  [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53982, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:14,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:14,760 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44223
2024-11-25T02:14:14,760 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:14,761 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:14,762 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42625
2024-11-25T02:14:14,762 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:14,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c00ed77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:14,763 DEBUG [Thread-123 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:14,764 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,769 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,769 DEBUG [NettyRpcServer-24-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,771 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:14,772 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,772 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:14,773 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:14,774 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,774 DEBUG [NettyRpcServer-24-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,774 DEBUG [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:14,774 INFO  [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:14,775 INFO  [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37616, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:14,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:14,789 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42625
2024-11-25T02:14:14,790 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:14,790 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:14,792 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38285
2024-11-25T02:14:14,792 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:14,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2849a3a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:14,794 DEBUG [Thread-124 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:14,795 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,797 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,797 DEBUG [NettyRpcServer-25-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,799 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:14,800 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,800 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:14,800 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,800 DEBUG [NettyRpcServer-25-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,800 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:14,801 DEBUG [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int
2024-11-25T02:14:14,801 INFO  [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:14,801 INFO  [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37240, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:14,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:14,827 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38285
2024-11-25T02:14:14,828 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:14,828 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:14,832 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45183
2024-11-25T02:14:14,833 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:14,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ec2847, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:14,834 DEBUG [Thread-125 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:14,835 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,837 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,837 DEBUG [NettyRpcServer-26-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,838 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:14,839 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,839 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:14,840 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,840 DEBUG [NettyRpcServer-26-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,840 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:14,840 DEBUG [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T02:14:14,840 INFO  [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:14,841 INFO  [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56548, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:14,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:14,851 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45183
2024-11-25T02:14:14,861 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=87 (was 76)
Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1640)
	java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-24-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-25-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-22-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-23-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-22-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-23-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-26-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-26-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-25-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-24-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=538 (was 493) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=738 (was 738), ProcessCount=11 (was 11), AvailableMemoryMB=3688 (was 3702)
2024-11-25T02:14:14,868 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=86, OpenFileDescriptor=538, MaxFileDescriptor=1048576, SystemLoadAverage=738, ProcessCount=11, AvailableMemoryMB=3688
2024-11-25T02:14:14,881 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:14,882 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:14,884 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500854884,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:14,886 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:14,887 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:14,888 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34873
2024-11-25T02:14:14,888 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:14,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eff3b2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:14,890 DEBUG [Thread-126 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-25T02:14:14,893 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(424): process preamble call response with response type null
2024-11-25T02:14:14,893 WARN  [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call
2024-11-25T02:14:14,893 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes
2024-11-25T02:14:14,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:14,894 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34873
2024-11-25T02:14:14,901 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=88 (was 86) - Thread LEAK? -, OpenFileDescriptor=547 (was 538) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=738 (was 738), ProcessCount=11 (was 11), AvailableMemoryMB=3686 (was 3688)
2024-11-25T02:14:14,907 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=88, OpenFileDescriptor=547, MaxFileDescriptor=1048576, SystemLoadAverage=738, ProcessCount=11, AvailableMemoryMB=3686
2024-11-25T02:14:14,918 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:14,919 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:14,921 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500854921,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:14,923 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:14,923 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:14,925 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43641
2024-11-25T02:14:14,925 WARN  [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! *********
2024-11-25T02:14:14,925 WARN  [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients
2024-11-25T02:14:14,925 WARN  [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true).
2024-11-25T02:14:14,925 WARN  [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user
2024-11-25T02:14:14,926 WARN  [Time-limited test {}] ipc.RpcServer(380): impersonation is possible!
2024-11-25T02:14:14,926 WARN  [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible,
2024-11-25T02:14:14,926 WARN  [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml
2024-11-25T02:14:14,926 WARN  [Time-limited test {}] ipc.RpcServer(383): ****************************
2024-11-25T02:14:14,926 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:14,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b6d1847, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:14,927 DEBUG [Thread-127 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-25T02:14:14,930 WARN  [NettyRpcServer-28-2 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1
2024-11-25T02:14:14,930 INFO  [NettyRpcServer-28-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-25T02:14:14,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:14,939 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43641
2024-11-25T02:14:14,947 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=90 (was 88) - Thread LEAK? -, OpenFileDescriptor=556 (was 547) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=738 (was 738), ProcessCount=11 (was 11), AvailableMemoryMB=3683 (was 3686)
2024-11-25T02:14:14,954 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=90, OpenFileDescriptor=556, MaxFileDescriptor=1048576, SystemLoadAverage=738, ProcessCount=11, AvailableMemoryMB=3683
2024-11-25T02:14:14,968 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:14,968 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:14,970 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500854970,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:14,972 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:14,973 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:14,984 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40085
2024-11-25T02:14:14,985 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:14,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ce6dcf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:14,986 DEBUG [Thread-128 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:14,987 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,990 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500854990,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,991 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,992 DEBUG [NettyRpcServer-29-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,993 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:14,994 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,994 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:14,994 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:14,994 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:14,995 DEBUG [NettyRpcServer-29-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:14,995 DEBUG [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:14,995 INFO  [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:14,995 INFO  [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33252, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:15,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:15,003 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40085
2024-11-25T02:14:15,012 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=92 (was 90) - Thread LEAK? -, OpenFileDescriptor=565 (was 556) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=738 (was 738), ProcessCount=11 (was 11), AvailableMemoryMB=3678 (was 3683)
2024-11-25T02:14:15,024 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=92, OpenFileDescriptor=565, MaxFileDescriptor=1048576, SystemLoadAverage=738, ProcessCount=11, AvailableMemoryMB=3678
2024-11-25T02:14:15,036 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:15,036 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:15,038 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500855038,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:15,040 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:15,041 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:15,042 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40193
2024-11-25T02:14:15,042 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:15,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c53df9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:15,043 DEBUG [Thread-129 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:15,044 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:15,047 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500855047,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:15,049 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:15,049 DEBUG [NettyRpcServer-30-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:15,050 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:15,051 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:15,051 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:15,052 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:15,052 DEBUG [NettyRpcServer-30-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:15,052 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:15,052 DEBUG [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T02:14:15,052 INFO  [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:15,053 INFO  [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42630, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:15,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:15,061 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40193
2024-11-25T02:14:15,062 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:15,063 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:15,064 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46219
2024-11-25T02:14:15,064 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:15,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@467f7c28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:15,069 DEBUG [Thread-130 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:15,072 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:15,074 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:15,074 DEBUG [NettyRpcServer-31-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:15,077 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:15,078 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:15,078 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:15,079 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:15,079 DEBUG [NettyRpcServer-31-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:15,079 DEBUG [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T02:14:15,079 INFO  [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:15,080 INFO  [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37060, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:17,080 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T02:14:17,081 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:17,081 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46219
2024-11-25T02:14:17,081 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.FailedServers(52): Added failed server with address localhost:46219 to list caused by java.io.IOException: org.apache.hbase.thirdparty.io.netty.handler.timeout.ReadTimeoutException
2024-11-25T02:14:17,087 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=96 (was 92)
Potentially hanging thread: NettyRpcServer-31-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-30-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-31-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-30-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=583 (was 565) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=738 (was 738), ProcessCount=11 (was 11), AvailableMemoryMB=3672 (was 3678)
2024-11-25T02:14:17,093 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=96, OpenFileDescriptor=583, MaxFileDescriptor=1048576, SystemLoadAverage=738, ProcessCount=11, AvailableMemoryMB=3671
2024-11-25T02:14:17,104 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:17,104 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:17,106 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500857106,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:17,108 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:17,108 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:17,109 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45999
2024-11-25T02:14:17,109 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:17,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fbb6817, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:17,110 DEBUG [Thread-131 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:17,111 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:17,113 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500857113,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:17,115 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:17,115 DEBUG [NettyRpcServer-32-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:17,116 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:17,116 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:17,116 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:17,117 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:17,117 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:17,117 DEBUG [NettyRpcServer-32-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:17,117 DEBUG [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:17,117 INFO  [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:17,118 INFO  [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57224, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:17,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:17,123 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45999
2024-11-25T02:14:17,130 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=98 (was 96) - Thread LEAK? -, OpenFileDescriptor=592 (was 583) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=738 (was 738), ProcessCount=11 (was 11), AvailableMemoryMB=3670 (was 3671)
2024-11-25T02:14:17,132 WARN  [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732500857107
2024-11-25T02:14:17,135 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=98, OpenFileDescriptor=592, MaxFileDescriptor=1048576, SystemLoadAverage=738, ProcessCount=11, AvailableMemoryMB=3670
2024-11-25T02:14:17,144 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:17,144 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:17,146 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500857146,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:17,147 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:17,148 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:17,149 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44705
2024-11-25T02:14:17,149 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:17,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2677e937, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:17,150 DEBUG [Thread-132 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:17,151 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:17,153 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500857153,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:17,154 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:17,155 DEBUG [NettyRpcServer-33-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:17,156 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T02:14:17,156 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:17,156 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T02:14:17,157 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T02:14:17,157 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:17,157 DEBUG [NettyRpcServer-33-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:17,157 DEBUG [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T02:14:17,157 INFO  [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T02:14:17,157 INFO  [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40214, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T02:14:17,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:17,163 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44705
2024-11-25T02:14:17,171 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=100 (was 98) - Thread LEAK? -, OpenFileDescriptor=601 (was 592) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=738 (was 738), ProcessCount=11 (was 11), AvailableMemoryMB=3668 (was 3670)
2024-11-25T02:14:17,179 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=100, OpenFileDescriptor=601, MaxFileDescriptor=1048576, SystemLoadAverage=738, ProcessCount=11, AvailableMemoryMB=3667
2024-11-25T02:14:17,189 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T02:14:17,189 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T02:14:17,191 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732500857191,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T02:14:17,193 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T02:14:17,193 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T02:14:17,194 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37191
2024-11-25T02:14:17,194 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T02:14:17,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c5ef0d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T02:14:17,195 DEBUG [Thread-133 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T02:14:17,196 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:17,198 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732500857198,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T02:14:17,199 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T02:14:17,200 INFO  [NettyRpcServer-34-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-25T02:14:17,206 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T02:14:17,206 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37191
2024-11-25T02:14:17,212 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=102 (was 100) - Thread LEAK? -, OpenFileDescriptor=610 (was 601) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=738 (was 738), ProcessCount=11 (was 11), AvailableMemoryMB=3665 (was 3667)
2024-11-25T02:14:17,213 INFO  [Time-limited test {}] impl.DefaultInternalKdcServerImpl(102): Default Internal kdc server stopped.