2024-11-08 08:14:25,263 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 2024-11-08 08:14:25,280 main DEBUG Took 0.014454 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 08:14:25,280 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 08:14:25,281 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 08:14:25,282 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 08:14:25,284 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 08:14:25,297 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 08:14:25,315 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 08:14:25,316 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 08:14:25,317 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 08:14:25,317 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 08:14:25,318 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 08:14:25,318 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 08:14:25,319 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 08:14:25,319 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 08:14:25,320 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 08:14:25,320 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 08:14:25,321 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 08:14:25,321 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 08:14:25,322 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 08:14:25,322 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 08:14:25,323 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 08:14:25,323 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 08:14:25,324 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 08:14:25,324 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 08:14:25,324 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 08:14:25,325 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 08:14:25,325 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 08:14:25,325 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 08:14:25,326 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 08:14:25,326 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 08:14:25,326 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 08:14:25,327 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 08:14:25,328 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 08:14:25,330 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 08:14:25,331 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 08:14:25,332 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 08:14:25,333 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 08:14:25,334 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 08:14:25,344 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 08:14:25,348 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 08:14:25,351 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 08:14:25,351 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 08:14:25,352 main DEBUG createAppenders(={Console}) 2024-11-08 08:14:25,353 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 initialized 2024-11-08 08:14:25,353 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 2024-11-08 08:14:25,354 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 OK. 2024-11-08 08:14:25,355 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 08:14:25,355 main DEBUG OutputStream closed 2024-11-08 08:14:25,356 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 08:14:25,356 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 08:14:25,356 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5d0a1059 OK 2024-11-08 08:14:25,433 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 08:14:25,436 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 08:14:25,438 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 08:14:25,439 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 08:14:25,440 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 08:14:25,440 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 08:14:25,441 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 08:14:25,442 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 08:14:25,442 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 08:14:25,443 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 08:14:25,443 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 08:14:25,444 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 08:14:25,444 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 08:14:25,445 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 08:14:25,445 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 08:14:25,446 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 08:14:25,446 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 08:14:25,447 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 08:14:25,450 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 08:14:25,451 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5812f68b) with optional ClassLoader: null 2024-11-08 08:14:25,451 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 08:14:25,452 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5812f68b] started OK. 2024-11-08T08:14:25,775 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d595e9fb-7189-b06e-d806-3297ae030d73 2024-11-08 08:14:25,779 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 08:14:25,780 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T08:14:25,801 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.security.TestSecureIPC timeout: 52 mins 2024-11-08T08:14:26,026 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=13, OpenFileDescriptor=288, MaxFileDescriptor=1048576, SystemLoadAverage=931, ProcessCount=11, AvailableMemoryMB=6734 2024-11-08T08:14:26,120 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T08:14:26,201 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:26,206 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:26,261 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053666258,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:27,031 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:27,133 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T08:14:27,165 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T08:14:27,175 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T08:14:27,193 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39293 2024-11-08T08:14:27,198 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:27,198 INFO [Listener,port=39293 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39293: starting 2024-11-08T08:14:27,198 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:27,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@330fa5fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:27,329 DEBUG [Thread-6 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:27,389 DEBUG [Thread-6 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39293 2024-11-08T08:14:27,400 DEBUG [Thread-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:27,450 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053667448,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:27,459 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:27,470 DEBUG [Reader=1,bindAddress=localhost,port=39293 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:27,476 DEBUG [Reader=1,bindAddress=localhost,port=39293 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:27,477 DEBUG [Reader=1,bindAddress=localhost,port=39293 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:27,486 DEBUG [Reader=1,bindAddress=localhost,port=39293 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:27,488 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:27,489 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:27,490 DEBUG [Reader=1,bindAddress=localhost,port=39293 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:27,490 DEBUG [Reader=1,bindAddress=localhost,port=39293 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:27,492 DEBUG [Reader=1,bindAddress=localhost,port=39293 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:27,492 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:27,493 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:27,494 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-08T08:14:27,494 DEBUG [Reader=1,bindAddress=localhost,port=39293 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:27,494 DEBUG [Reader=1,bindAddress=localhost,port=39293 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:27,494 DEBUG [Thread-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:27,495 DEBUG [Reader=1,bindAddress=localhost,port=39293 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:27,495 DEBUG [Reader=1,bindAddress=localhost,port=39293 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-08T08:14:27,496 INFO [Reader=1,bindAddress=localhost,port=39293 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:27,520 INFO [Reader=1,bindAddress=localhost,port=39293 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55666, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:27,520 DEBUG [Thread-6 {}] ipc.BlockingRpcConnection(634): Length of response for connection header:93 2024-11-08T08:14:27,534 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:250 2024-11-08T08:14:27,537 DEBUG [BRPC Connection (654492810) to localhost:39293 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-08T08:14:27,571 DEBUG [BRPC Connection (654492810) to localhost:39293 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:236 2024-11-08T08:14:27,577 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:2258 2024-11-08T08:14:27,582 DEBUG [BRPC Connection (654492810) to localhost:39293 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-08T08:14:27,586 DEBUG [BRPC Connection (654492810) to localhost:39293 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:2244 2024-11-08T08:14:27,602 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:4117 2024-11-08T08:14:27,604 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:18206 2024-11-08T08:14:27,610 DEBUG [BRPC Connection (654492810) to localhost:39293 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-08T08:14:27,629 DEBUG [BRPC Connection (654492810) to localhost:39293 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:22309 2024-11-08T08:14:27,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:27,637 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39293 2024-11-08T08:14:27,638 INFO [Listener,port=39293 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39293: stopping 2024-11-08T08:14:27,638 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:27,639 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:27,650 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 13) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: pool-3-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=290 (was 288) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=929 (was 931), ProcessCount=11 (was 11), AvailableMemoryMB=6585 (was 6734) 2024-11-08T08:14:27,658 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=929, ProcessCount=11, AvailableMemoryMB=6585 2024-11-08T08:14:27,674 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:27,675 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:27,684 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053667683,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:27,704 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:27,707 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=38337 2024-11-08T08:14:27,708 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:27,708 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:27,709 INFO [Listener,port=38337 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=38337: starting 2024-11-08T08:14:27,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ef429cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-08T08:14:27,713 DEBUG [Thread-11 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:27,714 DEBUG [Thread-11 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:38337 2024-11-08T08:14:27,716 DEBUG [Thread-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:27,729 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053667728,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:27,734 DEBUG [Thread-11 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:27,735 DEBUG [Thread-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-08T08:14:27,736 DEBUG [Thread-11 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:38337 org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections. at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:111) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-08T08:14:27,741 WARN [Thread-11 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731053667687 2024-11-08T08:14:27,806 DEBUG [Thread-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:27,809 DEBUG [Thread-11 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:27,809 DEBUG [Thread-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-08T08:14:27,810 DEBUG [Thread-11 {}] ipc.FailedServers(52): Added failed server with address localhost:38337 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:27,810 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:27,811 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 38337 2024-11-08T08:14:27,811 INFO [Listener,port=38337 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=38337: stopping 2024-11-08T08:14:27,811 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:27,811 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:27,816 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:383) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:375) app//org.apache.logging.log4j.core.layout.PatternLayout.toSerializable(PatternLayout.java:216) app//org.apache.logging.log4j.core.layout.PatternLayout.toSerializable(PatternLayout.java:59) app//org.apache.logging.log4j.core.layout.AbstractStringLayout.toByteArray(AbstractStringLayout.java:294) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:145) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=929 (was 929), ProcessCount=11 (was 11), AvailableMemoryMB=6572 (was 6585) 2024-11-08T08:14:27,822 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=929, ProcessCount=11, AvailableMemoryMB=6572 2024-11-08T08:14:27,834 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:27,834 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:27,840 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053667839,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:27,845 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:27,846 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=33267 2024-11-08T08:14:27,847 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:27,847 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:27,847 INFO [Listener,port=33267 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=33267: starting 2024-11-08T08:14:27,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b6e1dd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:27,849 DEBUG [Thread-14 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:27,849 DEBUG [Thread-14 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:33267 2024-11-08T08:14:27,851 DEBUG [Thread-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:27,860 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053667859,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:27,863 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:27,863 DEBUG [Reader=1,bindAddress=localhost,port=33267 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:27,865 DEBUG [Reader=1,bindAddress=localhost,port=33267 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:27,865 DEBUG [Reader=1,bindAddress=localhost,port=33267 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:27,867 DEBUG [Reader=1,bindAddress=localhost,port=33267 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:27,867 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:27,868 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:27,868 DEBUG [Reader=1,bindAddress=localhost,port=33267 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:27,868 DEBUG [Reader=1,bindAddress=localhost,port=33267 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:27,869 DEBUG [Reader=1,bindAddress=localhost,port=33267 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:27,869 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:27,869 DEBUG [Thread-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-08T08:14:27,870 DEBUG [Thread-14 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:33267 javax.security.sasl.SaslException: No common protection layer between client and server at com.sun.security.sasl.gsskerb.GssKrb5Client.doFinalHandshake(GssKrb5Client.java:270) ~[jdk.security.jgss:?] at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:203) ~[jdk.security.jgss:?] at org.apache.hadoop.hbase.security.AbstractHBaseSaslRpcClient.evaluateChallenge(AbstractHBaseSaslRpcClient.java:105) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:126) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-08T08:14:27,870 WARN [Thread-14 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731053667842 2024-11-08T08:14:27,970 DEBUG [Thread-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:27,976 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:27,976 DEBUG [Reader=2,bindAddress=localhost,port=33267 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:27,979 DEBUG [Reader=2,bindAddress=localhost,port=33267 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:27,979 DEBUG [Reader=2,bindAddress=localhost,port=33267 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:27,982 DEBUG [Reader=2,bindAddress=localhost,port=33267 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:27,982 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:27,983 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:27,983 DEBUG [Reader=2,bindAddress=localhost,port=33267 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:27,983 DEBUG [Reader=2,bindAddress=localhost,port=33267 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:27,983 DEBUG [Reader=2,bindAddress=localhost,port=33267 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:27,984 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:27,984 DEBUG [Thread-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-08T08:14:27,984 DEBUG [Thread-14 {}] ipc.FailedServers(52): Added failed server with address localhost:33267 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:27,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:27,985 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 33267 2024-11-08T08:14:27,985 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:27,985 INFO [Listener,port=33267 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=33267: stopping 2024-11-08T08:14:27,985 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:27,990 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=21 (was 19) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:615) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) Potentially hanging thread: RpcServer idle connection scanner for port 33267 java.base@17.0.11/java.util.TaskQueue.clear(Timer.java:678) java.base@17.0.11/java.util.TimerThread.run(Timer.java:521) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=929 (was 929), ProcessCount=11 (was 11), AvailableMemoryMB=6560 (was 6572) 2024-11-08T08:14:27,995 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=929, ProcessCount=11, AvailableMemoryMB=6560 2024-11-08T08:14:28,008 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:28,008 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:28,013 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053668013,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:28,018 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:28,020 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=46093 2024-11-08T08:14:28,021 INFO [Listener,port=46093 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=46093: starting 2024-11-08T08:14:28,021 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:28,021 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:28,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@614e09a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:28,023 DEBUG [Thread-17 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:28,024 DEBUG [Thread-17 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:46093 2024-11-08T08:14:28,025 DEBUG [Thread-17 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,032 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053668032,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,035 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:28,035 DEBUG [Reader=1,bindAddress=localhost,port=46093 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,037 DEBUG [Reader=1,bindAddress=localhost,port=46093 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,037 DEBUG [Reader=1,bindAddress=localhost,port=46093 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,039 DEBUG [Reader=1,bindAddress=localhost,port=46093 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:28,040 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:28,040 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:28,041 DEBUG [Reader=1,bindAddress=localhost,port=46093 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,041 DEBUG [Reader=1,bindAddress=localhost,port=46093 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,041 DEBUG [Reader=1,bindAddress=localhost,port=46093 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:28,041 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:28,042 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:28,042 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-08T08:14:28,042 DEBUG [Thread-17 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:28,042 DEBUG [Reader=1,bindAddress=localhost,port=46093 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,042 DEBUG [Reader=1,bindAddress=localhost,port=46093 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,043 DEBUG [Reader=1,bindAddress=localhost,port=46093 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,043 DEBUG [Reader=1,bindAddress=localhost,port=46093 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:28,043 INFO [Reader=1,bindAddress=localhost,port=46093 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:28,044 INFO [Reader=1,bindAddress=localhost,port=46093 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43564, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:28,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:28,076 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 46093 2024-11-08T08:14:28,077 INFO [Listener,port=46093 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=46093: stopping 2024-11-08T08:14:28,077 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:28,077 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:28,078 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:28,079 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41195 2024-11-08T08:14:28,080 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:28,080 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:28,080 INFO [Listener,port=41195 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41195: starting 2024-11-08T08:14:28,081 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7383bfdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:28,081 DEBUG [Thread-20 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:28,082 DEBUG [Thread-20 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:41195 2024-11-08T08:14:28,083 DEBUG [Thread-20 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,085 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:28,085 DEBUG [Reader=1,bindAddress=localhost,port=41195 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,086 DEBUG [Reader=1,bindAddress=localhost,port=41195 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,086 DEBUG [Reader=1,bindAddress=localhost,port=41195 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,087 DEBUG [Reader=1,bindAddress=localhost,port=41195 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:28,088 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:28,088 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:28,088 DEBUG [Reader=1,bindAddress=localhost,port=41195 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,089 DEBUG [Reader=1,bindAddress=localhost,port=41195 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,089 DEBUG [Reader=1,bindAddress=localhost,port=41195 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:28,089 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:28,090 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:28,090 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-08T08:14:28,090 DEBUG [Reader=1,bindAddress=localhost,port=41195 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,090 DEBUG [Thread-20 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:28,090 DEBUG [Reader=1,bindAddress=localhost,port=41195 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,090 DEBUG [Reader=1,bindAddress=localhost,port=41195 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,091 DEBUG [Reader=1,bindAddress=localhost,port=41195 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:28,091 INFO [Reader=1,bindAddress=localhost,port=41195 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:28,091 INFO [Reader=1,bindAddress=localhost,port=41195 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:52284, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:28,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:28,119 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41195 2024-11-08T08:14:28,119 INFO [Listener,port=41195 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41195: stopping 2024-11-08T08:14:28,119 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:28,120 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:28,120 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:28,122 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=46589 2024-11-08T08:14:28,123 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:28,123 INFO [Listener,port=46589 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=46589: starting 2024-11-08T08:14:28,123 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:28,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aed89c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:28,124 DEBUG [Thread-23 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:28,124 DEBUG [Thread-23 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:46589 2024-11-08T08:14:28,125 DEBUG [Thread-23 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,127 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:28,127 DEBUG [Reader=1,bindAddress=localhost,port=46589 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,129 DEBUG [Reader=1,bindAddress=localhost,port=46589 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,129 DEBUG [Reader=1,bindAddress=localhost,port=46589 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,131 DEBUG [Reader=1,bindAddress=localhost,port=46589 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:28,131 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:28,132 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:28,132 DEBUG [Reader=1,bindAddress=localhost,port=46589 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,132 DEBUG [Reader=1,bindAddress=localhost,port=46589 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,132 DEBUG [Reader=1,bindAddress=localhost,port=46589 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:28,133 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:28,133 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:28,133 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-08T08:14:28,133 DEBUG [Reader=1,bindAddress=localhost,port=46589 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,133 DEBUG [Thread-23 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:28,133 DEBUG [Reader=1,bindAddress=localhost,port=46589 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,134 DEBUG [Reader=1,bindAddress=localhost,port=46589 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,134 DEBUG [Reader=1,bindAddress=localhost,port=46589 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:28,134 INFO [Reader=1,bindAddress=localhost,port=46589 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:28,134 INFO [Reader=1,bindAddress=localhost,port=46589 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53930, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:28,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:28,152 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 46589 2024-11-08T08:14:28,152 INFO [Listener,port=46589 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=46589: stopping 2024-11-08T08:14:28,152 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:28,152 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:28,153 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:28,154 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=38425 2024-11-08T08:14:28,155 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:28,156 INFO [Listener,port=38425 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=38425: starting 2024-11-08T08:14:28,156 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:28,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eb38ce8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:28,158 DEBUG [Thread-26 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:28,158 DEBUG [Thread-26 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:38425 2024-11-08T08:14:28,160 DEBUG [Thread-26 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,162 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:28,162 DEBUG [Reader=1,bindAddress=localhost,port=38425 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,164 DEBUG [Reader=1,bindAddress=localhost,port=38425 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,164 DEBUG [Reader=1,bindAddress=localhost,port=38425 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,165 DEBUG [Reader=1,bindAddress=localhost,port=38425 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:28,166 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:28,166 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:28,166 DEBUG [Reader=1,bindAddress=localhost,port=38425 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,167 DEBUG [Reader=1,bindAddress=localhost,port=38425 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,167 DEBUG [Reader=1,bindAddress=localhost,port=38425 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:28,167 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:28,168 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:28,168 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-int 2024-11-08T08:14:28,168 DEBUG [Reader=1,bindAddress=localhost,port=38425 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,168 DEBUG [Thread-26 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:28,168 DEBUG [Reader=1,bindAddress=localhost,port=38425 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,168 DEBUG [Reader=1,bindAddress=localhost,port=38425 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,169 DEBUG [Reader=1,bindAddress=localhost,port=38425 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int 2024-11-08T08:14:28,169 INFO [Reader=1,bindAddress=localhost,port=38425 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:28,170 INFO [Reader=1,bindAddress=localhost,port=38425 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:45576, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:28,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:28,181 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 38425 2024-11-08T08:14:28,182 INFO [Listener,port=38425 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=38425: stopping 2024-11-08T08:14:28,182 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:28,182 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:28,182 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:28,183 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39491 2024-11-08T08:14:28,184 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:28,184 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:28,184 INFO [Listener,port=39491 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39491: starting 2024-11-08T08:14:28,185 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300cf047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:28,186 DEBUG [Thread-29 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:28,186 DEBUG [Thread-29 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39491 2024-11-08T08:14:28,187 DEBUG [Thread-29 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,188 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:28,189 DEBUG [Reader=1,bindAddress=localhost,port=39491 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,190 DEBUG [Reader=1,bindAddress=localhost,port=39491 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,190 DEBUG [Reader=1,bindAddress=localhost,port=39491 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,191 DEBUG [Reader=1,bindAddress=localhost,port=39491 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:28,191 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:28,192 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:28,192 DEBUG [Reader=1,bindAddress=localhost,port=39491 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,192 DEBUG [Reader=1,bindAddress=localhost,port=39491 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,192 DEBUG [Reader=1,bindAddress=localhost,port=39491 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:28,192 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:28,193 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:28,193 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-08T08:14:28,193 DEBUG [Reader=1,bindAddress=localhost,port=39491 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,193 DEBUG [Thread-29 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:28,193 DEBUG [Reader=1,bindAddress=localhost,port=39491 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,193 DEBUG [Reader=1,bindAddress=localhost,port=39491 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,193 DEBUG [Reader=1,bindAddress=localhost,port=39491 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-08T08:14:28,193 INFO [Reader=1,bindAddress=localhost,port=39491 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:28,194 INFO [Reader=1,bindAddress=localhost,port=39491 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47736, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:28,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:28,212 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39491 2024-11-08T08:14:28,212 INFO [Listener,port=39491 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39491: stopping 2024-11-08T08:14:28,212 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:28,213 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:28,219 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=22 (was 19) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeBytes(OutputStreamManager.java:353) app//org.apache.logging.log4j.core.layout.TextEncoderHelper.writeEncodedText(TextEncoderHelper.java:96) app//org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeText(TextEncoderHelper.java:65) app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:68) app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:32) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:227) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) Potentially hanging thread: Listener,port=39491 app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:219) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:252) Potentially hanging thread: RpcServer idle connection scanner for port 39491 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=929 (was 929), ProcessCount=11 (was 11), AvailableMemoryMB=6543 (was 6560) 2024-11-08T08:14:28,224 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=929, ProcessCount=11, AvailableMemoryMB=6543 2024-11-08T08:14:28,235 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:28,235 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:28,240 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053668239,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:28,245 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:28,246 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44603 2024-11-08T08:14:28,247 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:28,247 INFO [Listener,port=44603 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44603: starting 2024-11-08T08:14:28,247 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:28,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@430bb3ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:28,249 DEBUG [Thread-32 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-08T08:14:28,250 DEBUG [Thread-32 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44603 2024-11-08T08:14:28,254 DEBUG [BRPC Connection (654492810) to localhost:44603 from testuser {}] ipc.RpcConnection(424): process preamble call response with response type null 2024-11-08T08:14:28,254 WARN [BRPC Connection (654492810) to localhost:44603 from testuser {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call 2024-11-08T08:14:28,257 DEBUG [BRPC Connection (654492810) to localhost:44603 from testuser {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes 2024-11-08T08:14:28,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:28,258 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44603 2024-11-08T08:14:28,258 INFO [Listener,port=44603 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44603: stopping 2024-11-08T08:14:28,259 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:28,259 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:28,268 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 19), OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=929 (was 929), ProcessCount=11 (was 11), AvailableMemoryMB=6535 (was 6543) 2024-11-08T08:14:28,274 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=929, ProcessCount=11, AvailableMemoryMB=6535 2024-11-08T08:14:28,286 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:28,287 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:28,292 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053668291,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:28,295 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:28,297 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=33617 2024-11-08T08:14:28,297 WARN [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! ********* 2024-11-08T08:14:28,297 WARN [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients 2024-11-08T08:14:28,298 WARN [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true). 2024-11-08T08:14:28,298 WARN [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user 2024-11-08T08:14:28,298 WARN [Time-limited test {}] ipc.RpcServer(380): impersonation is possible! 2024-11-08T08:14:28,298 WARN [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible, 2024-11-08T08:14:28,298 WARN [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml 2024-11-08T08:14:28,298 WARN [Time-limited test {}] ipc.RpcServer(383): **************************** 2024-11-08T08:14:28,299 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:28,299 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:28,299 INFO [Listener,port=33617 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=33617: starting 2024-11-08T08:14:28,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2bf17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:28,300 DEBUG [Thread-35 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-08T08:14:28,300 DEBUG [Thread-35 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:33617 2024-11-08T08:14:28,301 WARN [Reader=1,bindAddress=localhost,port=33617 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1 2024-11-08T08:14:28,301 INFO [Reader=1,bindAddress=localhost,port=33617 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-08T08:14:28,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:28,310 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 33617 2024-11-08T08:14:28,310 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:28,311 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:28,312 INFO [Listener,port=33617 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=33617: stopping 2024-11-08T08:14:28,318 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=24 (was 19) Potentially hanging thread: Reader=0,bindAddress=localhost,port=33617 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Listener,port=33617 java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.fetchStackFrames(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.fetchStackFrames(StackStreamFactory.java:386) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.getNextBatch(StackStreamFactory.java:322) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.peekFrame(StackStreamFactory.java:263) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.hasNext(StackStreamFactory.java:351) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.tryAdvance(StackStreamFactory.java:593) java.base@17.0.11/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129) java.base@17.0.11/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527) java.base@17.0.11/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513) java.base@17.0.11/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499) java.base@17.0.11/java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150) java.base@17.0.11/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) java.base@17.0.11/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647) app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$15(StackLocator.java:100) app//org.apache.logging.log4j.util.StackLocator$$Lambda$125/0x00007f2d2418c260.apply(Unknown Source) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243) java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499) app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97) app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121) app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:252) Potentially hanging thread: RpcServer idle connection scanner for port 33617 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.io.BufferedOutputStream.write(BufferedOutputStream.java:118) app//org.apache.maven.surefire.api.util.internal.Channels$4.writeImpl(Channels.java:199) app//org.apache.maven.surefire.api.util.internal.AbstractNoninterruptibleWritableChannel.write(AbstractNoninterruptibleWritableChannel.java:66) app//org.apache.maven.surefire.api.util.internal.AbstractNoninterruptibleWritableChannel.writeBuffered(AbstractNoninterruptibleWritableChannel.java:49) app//org.apache.maven.surefire.api.stream.AbstractStreamEncoder.write(AbstractStreamEncoder.java:69) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.write(EventChannelEncoder.java:288) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.setOutErr(EventChannelEncoder.java:173) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.testOutput(EventChannelEncoder.java:168) app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:99) app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:43) app//org.apache.maven.surefire.common.junit4.JUnit4RunListener.writeTestOutput(JUnit4RunListener.java:202) app//org.apache.maven.surefire.api.report.ConsoleOutputCapture$ForwardingPrintStream.write(ConsoleOutputCapture.java:57) app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:250) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:283) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:294) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:217) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) Potentially hanging thread: FifoRpcScheduler.handler-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=929 (was 929), ProcessCount=11 (was 11), AvailableMemoryMB=6522 (was 6535) 2024-11-08T08:14:28,323 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=929, ProcessCount=11, AvailableMemoryMB=6522 2024-11-08T08:14:28,334 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:28,334 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:28,339 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053668339,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:28,343 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:28,344 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39375 2024-11-08T08:14:28,345 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:28,345 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:28,345 INFO [Listener,port=39375 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39375: starting 2024-11-08T08:14:28,346 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cdadc63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:28,347 DEBUG [Thread-38 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:28,347 DEBUG [Thread-38 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39375 2024-11-08T08:14:28,348 DEBUG [Thread-38 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,354 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053668353,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,356 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:28,356 DEBUG [Reader=1,bindAddress=localhost,port=39375 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,357 DEBUG [Reader=1,bindAddress=localhost,port=39375 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,357 DEBUG [Reader=1,bindAddress=localhost,port=39375 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,359 DEBUG [Reader=1,bindAddress=localhost,port=39375 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:28,359 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:28,359 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:28,360 DEBUG [Reader=1,bindAddress=localhost,port=39375 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,360 DEBUG [Reader=1,bindAddress=localhost,port=39375 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,360 DEBUG [Reader=1,bindAddress=localhost,port=39375 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:28,361 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:28,361 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:28,361 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-08T08:14:28,361 DEBUG [Reader=1,bindAddress=localhost,port=39375 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,361 DEBUG [Thread-38 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:28,361 DEBUG [Reader=1,bindAddress=localhost,port=39375 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,362 DEBUG [Reader=1,bindAddress=localhost,port=39375 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,362 DEBUG [Reader=1,bindAddress=localhost,port=39375 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:28,362 INFO [Reader=1,bindAddress=localhost,port=39375 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:28,362 INFO [Reader=1,bindAddress=localhost,port=39375 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44132, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:28,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:28,371 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39375 2024-11-08T08:14:28,371 INFO [Listener,port=39375 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39375: stopping 2024-11-08T08:14:28,371 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:28,372 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:28,377 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$14(StackLocator.java:99) app//org.apache.logging.log4j.util.StackLocator$$Lambda$127/0x00007f2d2418c700.test(Unknown Source) java.base@17.0.11/java.util.stream.WhileOps$1Op$1OpSink.accept(WhileOps.java:376) java.base@17.0.11/java.util.stream.WhileOps$1Op$1OpSink.accept(WhileOps.java:386) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.tryAdvance(StackStreamFactory.java:595) java.base@17.0.11/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129) java.base@17.0.11/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527) java.base@17.0.11/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513) java.base@17.0.11/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499) java.base@17.0.11/java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150) java.base@17.0.11/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) java.base@17.0.11/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647) app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$15(StackLocator.java:100) app//org.apache.logging.log4j.util.StackLocator$$Lambda$125/0x00007f2d2418c260.apply(Unknown Source) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243) java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499) app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97) app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121) app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=929 (was 929), ProcessCount=11 (was 11), AvailableMemoryMB=6507 (was 6522) 2024-11-08T08:14:28,382 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=929, ProcessCount=11, AvailableMemoryMB=6506 2024-11-08T08:14:28,393 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:28,393 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:28,398 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053668397,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:28,400 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:28,401 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34581 2024-11-08T08:14:28,402 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:28,402 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:28,402 INFO [Listener,port=34581 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34581: starting 2024-11-08T08:14:28,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@246e6cfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:28,403 DEBUG [Thread-41 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:28,404 DEBUG [Thread-41 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34581 2024-11-08T08:14:28,405 DEBUG [Thread-41 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,410 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053668410,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,412 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:28,413 DEBUG [Reader=1,bindAddress=localhost,port=34581 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,414 DEBUG [Reader=1,bindAddress=localhost,port=34581 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,414 DEBUG [Reader=1,bindAddress=localhost,port=34581 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,415 DEBUG [Reader=1,bindAddress=localhost,port=34581 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:28,415 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:28,416 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:28,416 DEBUG [Reader=1,bindAddress=localhost,port=34581 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,416 DEBUG [Reader=1,bindAddress=localhost,port=34581 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,416 DEBUG [Reader=1,bindAddress=localhost,port=34581 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:28,417 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:28,417 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:28,417 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-08T08:14:28,417 DEBUG [Reader=1,bindAddress=localhost,port=34581 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,417 DEBUG [Reader=1,bindAddress=localhost,port=34581 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,417 DEBUG [Thread-41 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:28,418 DEBUG [Reader=1,bindAddress=localhost,port=34581 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,418 DEBUG [Reader=1,bindAddress=localhost,port=34581 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-08T08:14:28,418 INFO [Reader=1,bindAddress=localhost,port=34581 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:28,418 INFO [Reader=1,bindAddress=localhost,port=34581 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39650, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:28,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:28,430 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34581 2024-11-08T08:14:28,430 INFO [Listener,port=34581 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34581: stopping 2024-11-08T08:14:28,430 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:28,431 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:28,431 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:28,432 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41217 2024-11-08T08:14:28,435 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:28,436 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:28,436 INFO [Listener,port=41217 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41217: starting 2024-11-08T08:14:28,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27af6aef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:28,438 DEBUG [Thread-44 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:28,438 DEBUG [Thread-44 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:41217 2024-11-08T08:14:28,439 DEBUG [Thread-44 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,440 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:28,441 DEBUG [Reader=1,bindAddress=localhost,port=41217 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,441 DEBUG [Reader=1,bindAddress=localhost,port=41217 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,441 DEBUG [Reader=1,bindAddress=localhost,port=41217 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,442 DEBUG [Reader=1,bindAddress=localhost,port=41217 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:28,443 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:28,443 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:28,443 DEBUG [Reader=1,bindAddress=localhost,port=41217 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,443 DEBUG [Reader=1,bindAddress=localhost,port=41217 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,443 DEBUG [Reader=1,bindAddress=localhost,port=41217 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:28,444 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:28,444 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:28,444 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-08T08:14:28,444 DEBUG [Reader=1,bindAddress=localhost,port=41217 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:28,444 DEBUG [Thread-44 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:28,444 DEBUG [Reader=1,bindAddress=localhost,port=41217 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:28,444 DEBUG [Reader=1,bindAddress=localhost,port=41217 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:28,444 DEBUG [Reader=1,bindAddress=localhost,port=41217 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-08T08:14:28,445 INFO [Reader=1,bindAddress=localhost,port=41217 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:28,445 INFO [Reader=1,bindAddress=localhost,port=41217 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59290, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:30,446 ERROR [Thread-44 {}] ipc.BlockingRpcConnection(646): Can't get the connection header response for rpc timeout, please check if server has the correct configuration to support the additional function. java.net.SocketTimeoutException: 2000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:59290 remote=localhost/127.0.0.1:41217] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:163) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:132) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:201) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.security.SaslInputStream.readMoreData(SaslInputStream.java:97) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.security.SaslInputStream.read(SaslInputStream.java:260) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:263) ~[?:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.processResponseForConnectionHeader(BlockingRpcConnection.java:630) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:570) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-08T08:14:30,448 DEBUG [Thread-44 {}] ipc.FailedServers(52): Added failed server with address localhost:41217 to list caused by java.io.IOException: Timeout while waiting connection header response 2024-11-08T08:14:30,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:30,449 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41217 2024-11-08T08:14:30,449 INFO [Listener,port=41217 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41217: stopping 2024-11-08T08:14:30,450 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:30,451 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:30,459 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=22 (was 19) Potentially hanging thread: Reader=2,bindAddress=localhost,port=41217 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Reader=0,bindAddress=localhost,port=41217 java.base@17.0.11/sun.nio.ch.FileDispatcherImpl.closeIntFD(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implClose(EPollSelectorImpl.java:216) java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:185) java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:151) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.hasNext(StackStreamFactory.java:351) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.tryAdvance(StackStreamFactory.java:593) java.base@17.0.11/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129) java.base@17.0.11/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527) java.base@17.0.11/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513) java.base@17.0.11/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499) java.base@17.0.11/java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150) java.base@17.0.11/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) java.base@17.0.11/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647) app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$15(StackLocator.java:100) app//org.apache.logging.log4j.util.StackLocator$$Lambda$125/0x00007f2d2418c260.apply(Unknown Source) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243) java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499) app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97) app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121) app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=929 (was 929), ProcessCount=11 (was 11), AvailableMemoryMB=6437 (was 6506) 2024-11-08T08:14:30,465 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=929, ProcessCount=11, AvailableMemoryMB=6436 2024-11-08T08:14:30,476 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:30,477 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:30,481 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053670481,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:30,484 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:30,486 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41033 2024-11-08T08:14:30,489 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:30,489 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:30,489 INFO [Listener,port=41033 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41033: starting 2024-11-08T08:14:30,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61edb8aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:30,491 DEBUG [Thread-47 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:30,491 DEBUG [Thread-47 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:41033 2024-11-08T08:14:30,492 DEBUG [Thread-47 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:30,499 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053670499,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:30,501 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:30,501 DEBUG [Reader=1,bindAddress=localhost,port=41033 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:30,502 DEBUG [Reader=1,bindAddress=localhost,port=41033 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:30,502 DEBUG [Reader=1,bindAddress=localhost,port=41033 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:30,503 DEBUG [Reader=1,bindAddress=localhost,port=41033 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:30,503 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:30,504 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:30,504 DEBUG [Reader=1,bindAddress=localhost,port=41033 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:30,504 DEBUG [Reader=1,bindAddress=localhost,port=41033 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:30,504 DEBUG [Reader=1,bindAddress=localhost,port=41033 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:30,504 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:30,505 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:30,505 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-08T08:14:30,505 DEBUG [Reader=1,bindAddress=localhost,port=41033 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:30,505 DEBUG [Thread-47 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:30,505 DEBUG [Reader=1,bindAddress=localhost,port=41033 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:30,505 DEBUG [Reader=1,bindAddress=localhost,port=41033 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:30,505 DEBUG [Reader=1,bindAddress=localhost,port=41033 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:30,505 INFO [Reader=1,bindAddress=localhost,port=41033 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:30,506 INFO [Reader=1,bindAddress=localhost,port=41033 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50440, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:30,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:30,514 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41033 2024-11-08T08:14:30,515 INFO [Listener,port=41033 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41033: stopping 2024-11-08T08:14:30,515 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:30,515 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:30,520 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=24 (was 19) Potentially hanging thread: Reader=2,bindAddress=localhost,port=41033 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243) java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499) app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97) app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121) app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) Potentially hanging thread: Reader=1,bindAddress=localhost,port=41033 java.base@17.0.11/sun.nio.ch.FileDispatcherImpl.closeIntFD(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implClose(EPollSelectorImpl.java:216) java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:185) java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:151) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer idle connection scanner for port 41033 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Listener,port=41033 java.base@17.0.11/java.util.TaskQueue.clear(Timer.java:675) java.base@17.0.11/java.util.Timer.cancel(Timer.java:442) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$ConnectionManager.stopIdleScan(SimpleRpcServer.java:633) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:265) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=929 (was 929), ProcessCount=11 (was 11), AvailableMemoryMB=6418 (was 6436) 2024-11-08T08:14:30,525 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=929, ProcessCount=11, AvailableMemoryMB=6417 2024-11-08T08:14:30,540 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:30,540 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:30,545 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053670545,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:30,549 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:30,551 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=37171 2024-11-08T08:14:30,551 INFO [Listener,port=37171 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=37171: starting 2024-11-08T08:14:30,551 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:30,551 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:30,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6083b989, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:30,553 DEBUG [Thread-50 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:30,553 DEBUG [Thread-50 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37171 2024-11-08T08:14:30,556 DEBUG [Thread-50 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:30,563 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053670563,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:30,566 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:30,566 DEBUG [Reader=1,bindAddress=localhost,port=37171 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:30,567 DEBUG [Reader=1,bindAddress=localhost,port=37171 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:30,567 DEBUG [Reader=1,bindAddress=localhost,port=37171 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:30,568 DEBUG [Reader=1,bindAddress=localhost,port=37171 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:30,569 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:30,569 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:30,569 DEBUG [Reader=1,bindAddress=localhost,port=37171 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:30,570 DEBUG [Reader=1,bindAddress=localhost,port=37171 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:30,570 DEBUG [Reader=1,bindAddress=localhost,port=37171 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:30,570 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:30,570 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:30,570 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-08T08:14:30,571 DEBUG [Reader=1,bindAddress=localhost,port=37171 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:30,571 DEBUG [Thread-50 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:30,571 DEBUG [Reader=1,bindAddress=localhost,port=37171 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:30,571 DEBUG [Reader=1,bindAddress=localhost,port=37171 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:30,571 DEBUG [Reader=1,bindAddress=localhost,port=37171 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:30,571 INFO [Reader=1,bindAddress=localhost,port=37171 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:30,571 INFO [Reader=1,bindAddress=localhost,port=37171 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41220, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:30,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:30,585 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 37171 2024-11-08T08:14:30,586 INFO [Listener,port=37171 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=37171: stopping 2024-11-08T08:14:30,586 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:30,586 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:30,592 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.spi.AbstractLogger.getRecursionDepthHolder(AbstractLogger.java:2166) app//org.apache.logging.log4j.spi.AbstractLogger.getRecursionDepth(AbstractLogger.java:2192) app//org.apache.logging.log4j.core.layout.AbstractStringLayout.getStringBuilder(AbstractStringLayout.java:129) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:225) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=929 (was 929), ProcessCount=11 (was 11), AvailableMemoryMB=6398 (was 6417) 2024-11-08T08:14:30,597 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=929, ProcessCount=11, AvailableMemoryMB=6397 2024-11-08T08:14:30,608 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:30,608 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:30,613 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053670613,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:30,616 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:30,618 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=33653 2024-11-08T08:14:30,620 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:30,621 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:30,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12b89fd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:30,622 INFO [Listener,port=33653 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=33653: starting 2024-11-08T08:14:30,622 DEBUG [Thread-53 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:30,623 DEBUG [Thread-53 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:33653 2024-11-08T08:14:30,624 DEBUG [Thread-53 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:30,630 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053670629,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:30,631 DEBUG [Thread-53 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:30,631 DEBUG [Thread-53 {}] security.HBaseSaslRpcClient(113): Server asks us to fall back to simple auth. 2024-11-08T08:14:30,631 DEBUG [Thread-53 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:30,631 INFO [Reader=1,bindAddress=localhost,port=33653 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-08T08:14:30,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:30,639 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 33653 2024-11-08T08:14:30,639 INFO [Listener,port=33653 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=33653: stopping 2024-11-08T08:14:30,639 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:30,639 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:30,645 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=22 (was 19) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeText(TextEncoderHelper.java:57) app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:68) app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:32) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:227) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) Potentially hanging thread: RpcServer idle connection scanner for port 33653 java.base@17.0.11/java.util.TimerThread.run(Timer.java:520) Potentially hanging thread: FifoRpcScheduler.handler-pool-0 java.base@17.0.11/java.util.IdentityHashMap$KeySet.iterator(IdentityHashMap.java:977) java.base@17.0.11/java.util.Collections$SetFromMap.iterator(Collections.java:5686) java.base@17.0.11/jdk.internal.misc.TerminatingThreadLocal.threadTerminated(TerminatingThreadLocal.java:69) java.base@17.0.11/java.lang.Thread.exit(Thread.java:850) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=929 (was 929), ProcessCount=11 (was 11), AvailableMemoryMB=6377 (was 6397) 2024-11-08T08:14:30,652 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=929, ProcessCount=11, AvailableMemoryMB=6376 2024-11-08T08:14:30,666 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:30,667 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:30,672 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053670672,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:30,676 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:30,689 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:30,692 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T08:14:30,759 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4584 (auto-detected) 2024-11-08T08:14:30,760 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-08T08:14:30,782 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42297 2024-11-08T08:14:30,782 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:30,782 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60ed97a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:30,783 DEBUG [Thread-54 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:30,784 DEBUG [Thread-54 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42297 2024-11-08T08:14:30,785 DEBUG [Thread-54 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:30,792 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053670792,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:30,794 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:30,822 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:30,822 DEBUG [NettyRpcServer-2-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:30,825 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:30,829 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:30,829 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:30,830 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:30,830 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:30,830 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:30,831 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:30,831 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-08T08:14:30,831 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:30,831 DEBUG [Thread-54 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:30,831 DEBUG [NettyRpcServer-2-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:30,831 DEBUG [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-08T08:14:30,831 INFO [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:30,836 INFO [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56888, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:30,836 DEBUG [Thread-54 {}] ipc.BlockingRpcConnection(634): Length of response for connection header:93 2024-11-08T08:14:30,837 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:251 2024-11-08T08:14:30,837 DEBUG [BRPC Connection (654492810) to localhost:42297 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-08T08:14:30,839 DEBUG [BRPC Connection (654492810) to localhost:42297 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:237 2024-11-08T08:14:30,840 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:2267 2024-11-08T08:14:30,840 DEBUG [BRPC Connection (654492810) to localhost:42297 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-08T08:14:30,841 DEBUG [BRPC Connection (654492810) to localhost:42297 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:2253 2024-11-08T08:14:30,843 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:4117 2024-11-08T08:14:30,844 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:18193 2024-11-08T08:14:30,844 DEBUG [BRPC Connection (654492810) to localhost:42297 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-08T08:14:30,855 DEBUG [BRPC Connection (654492810) to localhost:42297 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:22296 2024-11-08T08:14:30,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:30,855 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42297 2024-11-08T08:14:30,863 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=21 (was 19) Potentially hanging thread: NettyRpcServer-2-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-2-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=299 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=929 (was 929), ProcessCount=11 (was 11), AvailableMemoryMB=6331 (was 6376) 2024-11-08T08:14:30,868 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=21, OpenFileDescriptor=299, MaxFileDescriptor=1048576, SystemLoadAverage=929, ProcessCount=11, AvailableMemoryMB=6329 2024-11-08T08:14:30,878 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:30,878 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:30,882 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053670882,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:30,885 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:30,885 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:30,886 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34535 2024-11-08T08:14:30,886 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:30,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d38fc0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-08T08:14:30,887 DEBUG [Thread-55 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:30,887 DEBUG [Thread-55 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34535 2024-11-08T08:14:30,889 DEBUG [Thread-55 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:30,894 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053670894,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:30,896 DEBUG [Thread-55 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:30,896 DEBUG [Thread-55 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-08T08:14:30,896 DEBUG [Thread-55 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:34535 org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections. at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:111) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-08T08:14:30,897 WARN [Thread-55 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731053670883 2024-11-08T08:14:30,981 DEBUG [Thread-55 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:30,984 DEBUG [Thread-55 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:30,985 DEBUG [Thread-55 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-08T08:14:30,985 DEBUG [Thread-55 {}] ipc.FailedServers(52): Added failed server with address localhost:34535 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:30,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:30,986 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34535 2024-11-08T08:14:30,993 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=24 (was 21) Potentially hanging thread: NettyRpcServer-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=308 (was 299) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=929 (was 929), ProcessCount=11 (was 11), AvailableMemoryMB=6305 (was 6329) 2024-11-08T08:14:31,008 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=24, OpenFileDescriptor=308, MaxFileDescriptor=1048576, SystemLoadAverage=929, ProcessCount=11, AvailableMemoryMB=6305 2024-11-08T08:14:31,020 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:31,020 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:31,025 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053671025,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:31,028 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:31,028 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:31,029 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44025 2024-11-08T08:14:31,029 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:31,030 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b91de7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:31,030 DEBUG [Thread-56 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:31,031 DEBUG [Thread-56 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44025 2024-11-08T08:14:31,032 DEBUG [Thread-56 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,038 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053671037,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,039 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:31,039 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,039 DEBUG [NettyRpcServer-4-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,041 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:31,042 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:31,042 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:31,043 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,043 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:31,043 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:31,043 DEBUG [Thread-56 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-08T08:14:31,044 DEBUG [Thread-56 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:44025 javax.security.sasl.SaslException: No common protection layer between client and server at com.sun.security.sasl.gsskerb.GssKrb5Client.doFinalHandshake(GssKrb5Client.java:270) ~[jdk.security.jgss:?] at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:203) ~[jdk.security.jgss:?] at org.apache.hadoop.hbase.security.AbstractHBaseSaslRpcClient.evaluateChallenge(AbstractHBaseSaslRpcClient.java:105) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:126) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-08T08:14:31,044 WARN [Thread-56 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731053671026 2024-11-08T08:14:31,107 DEBUG [Thread-56 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,109 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:31,109 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,110 DEBUG [NettyRpcServer-4-3 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,111 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:31,112 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:31,112 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:31,113 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,113 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:31,113 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:31,113 DEBUG [Thread-56 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-08T08:14:31,114 DEBUG [Thread-56 {}] ipc.FailedServers(52): Added failed server with address localhost:44025 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:31,114 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44025 2024-11-08T08:14:31,119 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=27 (was 24) Potentially hanging thread: NettyRpcServer-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=317 (was 308) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=929 (was 929), ProcessCount=11 (was 11), AvailableMemoryMB=6278 (was 6305) 2024-11-08T08:14:31,124 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=27, OpenFileDescriptor=317, MaxFileDescriptor=1048576, SystemLoadAverage=929, ProcessCount=11, AvailableMemoryMB=6277 2024-11-08T08:14:31,137 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:31,137 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:31,141 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053671140,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:31,143 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:31,143 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:31,144 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43943 2024-11-08T08:14:31,144 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:31,145 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@573ddaab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:31,145 DEBUG [Thread-57 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:31,146 DEBUG [Thread-57 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43943 2024-11-08T08:14:31,147 DEBUG [Thread-57 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,153 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053671153,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,155 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:31,155 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,156 DEBUG [NettyRpcServer-5-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,157 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:31,158 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:31,158 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:31,159 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,159 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:31,160 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:31,160 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:31,160 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-08T08:14:31,160 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,160 DEBUG [Thread-57 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:31,160 DEBUG [NettyRpcServer-5-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,161 DEBUG [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:31,161 INFO [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:31,161 INFO [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34184, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:31,174 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:31,175 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43943 2024-11-08T08:14:31,176 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:31,177 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:31,178 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43037 2024-11-08T08:14:31,178 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:31,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67006bc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:31,179 DEBUG [Thread-58 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:31,180 DEBUG [Thread-58 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43037 2024-11-08T08:14:31,181 DEBUG [Thread-58 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,183 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:31,183 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,183 DEBUG [NettyRpcServer-6-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,185 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:31,185 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:31,186 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:31,186 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,186 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:31,186 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:31,187 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:31,187 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-08T08:14:31,187 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,187 DEBUG [Thread-58 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:31,187 DEBUG [NettyRpcServer-6-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,187 DEBUG [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:31,187 INFO [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:31,188 INFO [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56932, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:31,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:31,194 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43037 2024-11-08T08:14:31,194 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:31,195 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:31,196 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34383 2024-11-08T08:14:31,196 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:31,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36f67eb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:31,197 DEBUG [Thread-59 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:31,197 DEBUG [Thread-59 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34383 2024-11-08T08:14:31,198 DEBUG [Thread-59 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,199 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:31,199 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,200 DEBUG [NettyRpcServer-7-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,201 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:31,201 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:31,201 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:31,202 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,202 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:31,202 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:31,203 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:31,203 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-08T08:14:31,203 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,203 DEBUG [Thread-59 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:31,203 DEBUG [NettyRpcServer-7-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,203 DEBUG [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:31,203 INFO [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:31,204 INFO [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58452, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:31,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:31,211 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34383 2024-11-08T08:14:31,212 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:31,212 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:31,213 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35439 2024-11-08T08:14:31,213 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:31,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22085841, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:31,214 DEBUG [Thread-60 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:31,214 DEBUG [Thread-60 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:35439 2024-11-08T08:14:31,215 DEBUG [Thread-60 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,217 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:31,218 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,218 DEBUG [NettyRpcServer-8-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,220 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:31,221 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:31,222 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:31,222 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,222 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:31,223 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:31,223 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:31,223 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-int 2024-11-08T08:14:31,223 DEBUG [Thread-60 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:31,224 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,224 DEBUG [NettyRpcServer-8-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,224 DEBUG [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int 2024-11-08T08:14:31,224 INFO [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:31,225 INFO [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49554, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:31,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:31,237 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35439 2024-11-08T08:14:31,238 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:31,238 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:31,239 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40713 2024-11-08T08:14:31,239 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:31,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fdfbb53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:31,241 DEBUG [Thread-61 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:31,241 DEBUG [Thread-61 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40713 2024-11-08T08:14:31,242 DEBUG [Thread-61 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,243 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:31,243 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,243 DEBUG [NettyRpcServer-9-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,245 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:31,245 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:31,245 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:31,246 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,246 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:31,246 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:31,246 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:31,246 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-08T08:14:31,246 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,246 DEBUG [Thread-61 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:31,247 DEBUG [NettyRpcServer-9-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,247 DEBUG [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-08T08:14:31,247 INFO [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:31,248 INFO [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43708, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:31,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:31,259 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40713 2024-11-08T08:14:31,265 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=37 (was 27) Potentially hanging thread: NettyRpcServer-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=362 (was 317) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=929 (was 929), ProcessCount=11 (was 11), AvailableMemoryMB=6246 (was 6277) 2024-11-08T08:14:31,273 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=37, OpenFileDescriptor=362, MaxFileDescriptor=1048576, SystemLoadAverage=929, ProcessCount=11, AvailableMemoryMB=6234 2024-11-08T08:14:31,293 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:31,293 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:31,298 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053671297,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:31,300 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:31,301 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:31,305 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43033 2024-11-08T08:14:31,305 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:31,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ac88172, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:31,306 DEBUG [Thread-62 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-08T08:14:31,306 DEBUG [Thread-62 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43033 2024-11-08T08:14:31,308 DEBUG [BRPC Connection (654492810) to localhost:43033 from testuser {}] ipc.RpcConnection(424): process preamble call response with response type null 2024-11-08T08:14:31,308 WARN [BRPC Connection (654492810) to localhost:43033 from testuser {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call 2024-11-08T08:14:31,308 DEBUG [BRPC Connection (654492810) to localhost:43033 from testuser {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes 2024-11-08T08:14:31,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:31,308 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43033 2024-11-08T08:14:31,316 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=39 (was 37) Potentially hanging thread: NettyRpcServer-10-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-10-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=371 (was 362) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=929 (was 929), ProcessCount=11 (was 11), AvailableMemoryMB=6203 (was 6234) 2024-11-08T08:14:31,321 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=39, OpenFileDescriptor=371, MaxFileDescriptor=1048576, SystemLoadAverage=929, ProcessCount=11, AvailableMemoryMB=6203 2024-11-08T08:14:31,334 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:31,334 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:31,339 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053671338,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:31,342 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:31,342 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:31,343 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43419 2024-11-08T08:14:31,343 WARN [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! ********* 2024-11-08T08:14:31,343 WARN [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients 2024-11-08T08:14:31,343 WARN [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true). 2024-11-08T08:14:31,343 WARN [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user 2024-11-08T08:14:31,344 WARN [Time-limited test {}] ipc.RpcServer(380): impersonation is possible! 2024-11-08T08:14:31,344 WARN [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible, 2024-11-08T08:14:31,344 WARN [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml 2024-11-08T08:14:31,344 WARN [Time-limited test {}] ipc.RpcServer(383): **************************** 2024-11-08T08:14:31,344 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:31,344 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54511cec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:31,345 DEBUG [Thread-63 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-08T08:14:31,345 DEBUG [Thread-63 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43419 2024-11-08T08:14:31,347 WARN [NettyRpcServer-11-2 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1 2024-11-08T08:14:31,347 INFO [NettyRpcServer-11-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-08T08:14:31,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:31,353 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43419 2024-11-08T08:14:31,359 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=41 (was 39) Potentially hanging thread: NettyRpcServer-11-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-11-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=380 (was 371) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=929 (was 929), ProcessCount=11 (was 11), AvailableMemoryMB=6191 (was 6203) 2024-11-08T08:14:31,363 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=41, OpenFileDescriptor=380, MaxFileDescriptor=1048576, SystemLoadAverage=929, ProcessCount=11, AvailableMemoryMB=6191 2024-11-08T08:14:31,375 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:31,375 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:31,379 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053671379,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:31,381 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:31,381 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:31,382 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39077 2024-11-08T08:14:31,382 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:31,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44074ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:31,383 DEBUG [Thread-64 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:31,384 DEBUG [Thread-64 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39077 2024-11-08T08:14:31,384 DEBUG [Thread-64 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,390 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053671390,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,392 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:31,392 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,392 DEBUG [NettyRpcServer-12-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,393 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:31,394 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:31,394 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:31,394 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,395 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:31,395 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:31,395 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:31,395 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-08T08:14:31,395 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,396 DEBUG [Thread-64 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:31,396 DEBUG [NettyRpcServer-12-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,396 DEBUG [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:31,396 INFO [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:31,396 INFO [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:45010, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:31,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:31,403 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39077 2024-11-08T08:14:31,410 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=43 (was 41) Potentially hanging thread: NettyRpcServer-12-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-12-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=389 (was 380) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=929 (was 929), ProcessCount=11 (was 11), AvailableMemoryMB=6174 (was 6191) 2024-11-08T08:14:31,416 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=43, OpenFileDescriptor=389, MaxFileDescriptor=1048576, SystemLoadAverage=929, ProcessCount=11, AvailableMemoryMB=6173 2024-11-08T08:14:31,430 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:31,431 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:31,434 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053671434,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:31,437 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:31,437 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:31,438 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40451 2024-11-08T08:14:31,438 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:31,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13827bd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:31,440 DEBUG [Thread-65 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:31,440 DEBUG [Thread-65 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40451 2024-11-08T08:14:31,441 DEBUG [Thread-65 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,446 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053671446,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,448 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:31,448 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,448 DEBUG [NettyRpcServer-13-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,450 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:31,451 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:31,451 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:31,452 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,452 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:31,452 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:31,453 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:31,453 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-08T08:14:31,453 DEBUG [Thread-65 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:31,453 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,453 DEBUG [NettyRpcServer-13-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,453 DEBUG [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-08T08:14:31,453 INFO [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:31,454 INFO [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51090, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:31,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:31,465 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40451 2024-11-08T08:14:31,466 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:31,466 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:31,467 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33737 2024-11-08T08:14:31,467 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:31,467 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23688030, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:31,473 DEBUG [Thread-66 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:31,473 DEBUG [Thread-66 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:33737 2024-11-08T08:14:31,474 DEBUG [Thread-66 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,476 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:31,476 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,477 DEBUG [NettyRpcServer-14-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,479 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:31,479 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:31,480 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:31,480 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,480 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:31,480 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:31,481 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:31,481 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-08T08:14:31,481 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:31,481 DEBUG [Thread-66 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:31,481 DEBUG [NettyRpcServer-14-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:31,481 DEBUG [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-08T08:14:31,481 INFO [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:31,482 INFO [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47888, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:33,484 ERROR [Thread-66 {}] ipc.BlockingRpcConnection(646): Can't get the connection header response for rpc timeout, please check if server has the correct configuration to support the additional function. java.net.SocketTimeoutException: 2000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:47888 remote=localhost/127.0.0.1:33737] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:163) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:132) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:201) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.security.SaslInputStream.readMoreData(SaslInputStream.java:97) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.security.SaslInputStream.read(SaslInputStream.java:260) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:263) ~[?:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.processResponseForConnectionHeader(BlockingRpcConnection.java:630) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:570) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-08T08:14:33,484 DEBUG [Thread-66 {}] ipc.FailedServers(52): Added failed server with address localhost:33737 to list caused by java.io.IOException: Timeout while waiting connection header response 2024-11-08T08:14:33,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:33,485 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33737 2024-11-08T08:14:33,492 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=47 (was 43) Potentially hanging thread: NettyRpcServer-14-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-14-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-13-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-13-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=407 (was 389) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=886 (was 929), ProcessCount=11 (was 11), AvailableMemoryMB=6123 (was 6173) 2024-11-08T08:14:33,500 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=47, OpenFileDescriptor=407, MaxFileDescriptor=1048576, SystemLoadAverage=886, ProcessCount=11, AvailableMemoryMB=6122 2024-11-08T08:14:33,513 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:33,513 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:33,517 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053673517,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:33,519 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:33,520 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:33,520 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40843 2024-11-08T08:14:33,521 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:33,521 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@700210fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:33,522 DEBUG [Thread-67 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:33,522 DEBUG [Thread-67 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40843 2024-11-08T08:14:33,523 DEBUG [Thread-67 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,529 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053673529,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,531 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:33,531 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,531 DEBUG [NettyRpcServer-15-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,533 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:33,533 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:33,534 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:33,534 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,534 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:33,535 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:33,535 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:33,535 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-08T08:14:33,535 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,535 DEBUG [Thread-67 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:33,535 DEBUG [NettyRpcServer-15-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,535 DEBUG [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:33,535 INFO [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:33,536 INFO [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47792, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:33,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:33,544 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40843 2024-11-08T08:14:33,551 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=49 (was 47) Potentially hanging thread: NettyRpcServer-15-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-15-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=416 (was 407) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=886 (was 886), ProcessCount=11 (was 11), AvailableMemoryMB=6105 (was 6122) 2024-11-08T08:14:33,558 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=49, OpenFileDescriptor=416, MaxFileDescriptor=1048576, SystemLoadAverage=886, ProcessCount=11, AvailableMemoryMB=6104 2024-11-08T08:14:33,571 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:33,571 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:33,576 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053673575,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:33,579 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:33,579 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:33,581 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39211 2024-11-08T08:14:33,581 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:33,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@213d71b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:33,582 DEBUG [Thread-68 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:33,582 DEBUG [Thread-68 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39211 2024-11-08T08:14:33,583 DEBUG [Thread-68 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,587 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053673586,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,588 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:33,588 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,588 DEBUG [NettyRpcServer-16-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,589 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:33,590 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-08T08:14:33,590 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-08T08:14:33,590 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,590 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:33,590 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-08T08:14:33,590 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-08T08:14:33,591 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-08T08:14:33,591 DEBUG [Thread-68 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:33,591 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,591 DEBUG [NettyRpcServer-16-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,591 DEBUG [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:33,591 INFO [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:33,591 INFO [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:45086, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:33,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:33,598 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39211 2024-11-08T08:14:33,604 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=51 (was 49) Potentially hanging thread: NettyRpcServer-16-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-16-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=425 (was 416) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=886 (was 886), ProcessCount=11 (was 11), AvailableMemoryMB=6087 (was 6104) 2024-11-08T08:14:33,608 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=51, OpenFileDescriptor=425, MaxFileDescriptor=1048576, SystemLoadAverage=886, ProcessCount=11, AvailableMemoryMB=6087 2024-11-08T08:14:33,618 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:33,619 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:33,622 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053673622,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:33,625 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:33,626 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:33,627 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36679 2024-11-08T08:14:33,627 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:33,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c8e1f16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:33,628 DEBUG [Thread-69 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:33,629 DEBUG [Thread-69 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:36679 2024-11-08T08:14:33,629 DEBUG [Thread-69 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,633 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053673633,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,634 DEBUG [Thread-69 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-08T08:14:33,635 DEBUG [Thread-69 {}] security.HBaseSaslRpcClient(113): Server asks us to fall back to simple auth. 2024-11-08T08:14:33,635 DEBUG [Thread-69 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:33,635 INFO [NettyRpcServer-17-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-08T08:14:33,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:33,641 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36679 2024-11-08T08:14:33,647 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=53 (was 51) Potentially hanging thread: NettyRpcServer-17-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-17-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=434 (was 425) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=886 (was 886), ProcessCount=11 (was 11), AvailableMemoryMB=6070 (was 6087) 2024-11-08T08:14:33,653 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=53, OpenFileDescriptor=434, MaxFileDescriptor=1048576, SystemLoadAverage=886, ProcessCount=11, AvailableMemoryMB=6070 2024-11-08T08:14:33,663 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:33,664 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:33,667 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053673667,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:33,669 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:33,670 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45709 2024-11-08T08:14:33,670 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:33,671 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:33,671 INFO [Listener,port=45709 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45709: starting 2024-11-08T08:14:33,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10477af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:33,674 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T08:14:33,674 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T08:14:33,679 DEBUG [Thread-72 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:33,698 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,703 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053673703,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,705 DEBUG [Reader=1,bindAddress=localhost,port=45709 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,706 DEBUG [Reader=1,bindAddress=localhost,port=45709 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:33,706 DEBUG [Reader=1,bindAddress=localhost,port=45709 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,707 DEBUG [Reader=1,bindAddress=localhost,port=45709 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:33,708 DEBUG [Reader=1,bindAddress=localhost,port=45709 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:33,708 DEBUG [Reader=1,bindAddress=localhost,port=45709 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,708 DEBUG [Reader=1,bindAddress=localhost,port=45709 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:33,709 DEBUG [Reader=1,bindAddress=localhost,port=45709 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:33,709 DEBUG [Reader=1,bindAddress=localhost,port=45709 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,709 DEBUG [Reader=1,bindAddress=localhost,port=45709 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,709 DEBUG [Reader=1,bindAddress=localhost,port=45709 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-08T08:14:33,709 INFO [Reader=1,bindAddress=localhost,port=45709 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:33,716 INFO [Reader=1,bindAddress=localhost,port=45709 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49670, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:33,717 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:33,730 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:33,731 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45709 2024-11-08T08:14:33,731 INFO [Listener,port=45709 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45709: stopping 2024-11-08T08:14:33,731 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:33,731 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:33,740 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=55 (was 53) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-1 java.base@17.0.11/java.lang.ClassLoader.preDefineClass(ClassLoader.java:890) java.base@17.0.11/java.lang.ClassLoader.defineClass(ClassLoader.java:1015) java.base@17.0.11/java.security.SecureClassLoader.defineClass(SecureClassLoader.java:150) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.defineClass(BuiltinClassLoader.java:862) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:773) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:768) java.base@17.0.11/java.security.AccessController.executePrivileged(AccessController.java:776) java.base@17.0.11/java.security.AccessController.doPrivileged(AccessController.java:318) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.findClassOnClassPathOrNull(BuiltinClassLoader.java:781) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:681) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:639) java.base@17.0.11/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) java.base@17.0.11/java.lang.ClassLoader.loadClass(ClassLoader.java:525) app//org.apache.hadoop.hbase.ipc.IPCUtil$1.initialValue(IPCUtil.java:294) app//org.apache.hadoop.hbase.ipc.IPCUtil$1.initialValue(IPCUtil.java:290) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocal.initialize(FastThreadLocal.java:177) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocal.get(FastThreadLocal.java:142) app//org.apache.hadoop.hbase.ipc.IPCUtil.execute(IPCUtil.java:304) app//org.apache.hadoop.hbase.ipc.NettyRpcConnection.shutdown(NettyRpcConnection.java:146) app//org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:161) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) app//org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) app//org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Reader=1,bindAddress=localhost,port=45709 java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implDereg(EPollSelectorImpl.java:228) java.base@17.0.11/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:267) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:134) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 434) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=886 (was 886), ProcessCount=11 (was 11), AvailableMemoryMB=6058 (was 6070) 2024-11-08T08:14:33,746 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=54, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=886, ProcessCount=11, AvailableMemoryMB=6058 2024-11-08T08:14:33,756 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:33,757 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:33,760 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053673760,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:33,762 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:33,763 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=35601 2024-11-08T08:14:33,764 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:33,764 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:33,764 INFO [Listener,port=35601 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=35601: starting 2024-11-08T08:14:33,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@199ae87f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-08T08:14:33,765 DEBUG [Thread-75 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:33,767 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,771 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053673771,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,773 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-08T08:14:33,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:33,774 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 35601 2024-11-08T08:14:33,774 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.FailedServers(52): Added failed server with address localhost:35601 to list caused by org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections. 2024-11-08T08:14:33,774 INFO [Listener,port=35601 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=35601: stopping 2024-11-08T08:14:33,774 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:33,774 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:33,780 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=57 (was 54) Potentially hanging thread: Reader=2,bindAddress=localhost,port=35601 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-2 app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:345) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) app//org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Reader=0,bindAddress=localhost,port=35601 java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:258) java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:183) java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:151) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=886 (was 886), ProcessCount=11 (was 11), AvailableMemoryMB=6047 (was 6058) 2024-11-08T08:14:33,786 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=55, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=886, ProcessCount=11, AvailableMemoryMB=6047 2024-11-08T08:14:33,799 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:33,799 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:33,803 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053673803,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:33,805 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:33,806 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=38295 2024-11-08T08:14:33,806 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:33,806 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:33,806 INFO [Listener,port=38295 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=38295: starting 2024-11-08T08:14:33,806 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@587e9931, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:33,807 DEBUG [Thread-78 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:33,809 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,815 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053673815,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,818 DEBUG [Reader=1,bindAddress=localhost,port=38295 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,819 DEBUG [Reader=1,bindAddress=localhost,port=38295 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:33,819 DEBUG [Reader=1,bindAddress=localhost,port=38295 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,821 DEBUG [Reader=1,bindAddress=localhost,port=38295 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:33,822 DEBUG [Reader=1,bindAddress=localhost,port=38295 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:33,822 DEBUG [Reader=1,bindAddress=localhost,port=38295 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,822 DEBUG [Reader=1,bindAddress=localhost,port=38295 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:33,824 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-08T08:14:33,824 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.FailedServers(52): Added failed server with address localhost:38295 to list caused by javax.security.sasl.SaslException: No common protection layer between client and server 2024-11-08T08:14:33,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:33,825 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 38295 2024-11-08T08:14:33,825 INFO [Listener,port=38295 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=38295: stopping 2024-11-08T08:14:33,825 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:33,825 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:33,836 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=60 (was 55) Potentially hanging thread: Relogin-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Reader=1,bindAddress=localhost,port=38295 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Reader=0,bindAddress=localhost,port=38295 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer idle connection scanner for port 38295 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=886 (was 886), ProcessCount=11 (was 11), AvailableMemoryMB=6034 (was 6047) 2024-11-08T08:14:33,848 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=57, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=886, ProcessCount=11, AvailableMemoryMB=6034 2024-11-08T08:14:33,859 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:33,859 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:33,862 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053673862,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:33,865 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:33,866 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=43037 2024-11-08T08:14:33,867 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:33,867 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:33,867 INFO [Listener,port=43037 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=43037: starting 2024-11-08T08:14:33,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4899f97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:33,867 DEBUG [Thread-81 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:33,869 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,879 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053673879,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,881 DEBUG [Reader=1,bindAddress=localhost,port=43037 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,882 DEBUG [Reader=1,bindAddress=localhost,port=43037 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:33,882 DEBUG [Reader=1,bindAddress=localhost,port=43037 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,883 DEBUG [Reader=1,bindAddress=localhost,port=43037 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:33,883 DEBUG [Reader=1,bindAddress=localhost,port=43037 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:33,883 DEBUG [Reader=1,bindAddress=localhost,port=43037 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,883 DEBUG [Reader=1,bindAddress=localhost,port=43037 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:33,884 DEBUG [Reader=1,bindAddress=localhost,port=43037 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:33,884 DEBUG [Reader=1,bindAddress=localhost,port=43037 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,884 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:33,884 DEBUG [Reader=1,bindAddress=localhost,port=43037 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,884 DEBUG [Reader=1,bindAddress=localhost,port=43037 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:33,884 INFO [Reader=1,bindAddress=localhost,port=43037 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:33,885 INFO [Reader=1,bindAddress=localhost,port=43037 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46468, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:33,890 WARN [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731053673863 2024-11-08T08:14:33,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:33,890 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 43037 2024-11-08T08:14:33,890 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:33,890 INFO [Listener,port=43037 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=43037: stopping 2024-11-08T08:14:33,890 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:33,891 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:33,892 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34799 2024-11-08T08:14:33,892 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:33,892 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:33,892 INFO [Listener,port=34799 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34799: starting 2024-11-08T08:14:33,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a1e5b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:33,893 DEBUG [Thread-84 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:33,894 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,896 DEBUG [Reader=1,bindAddress=localhost,port=34799 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,896 DEBUG [Reader=1,bindAddress=localhost,port=34799 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:33,897 DEBUG [Reader=1,bindAddress=localhost,port=34799 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,897 DEBUG [Reader=1,bindAddress=localhost,port=34799 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:33,898 DEBUG [Reader=1,bindAddress=localhost,port=34799 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:33,898 DEBUG [Reader=1,bindAddress=localhost,port=34799 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,898 DEBUG [Reader=1,bindAddress=localhost,port=34799 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:33,898 DEBUG [Reader=1,bindAddress=localhost,port=34799 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:33,898 DEBUG [Reader=1,bindAddress=localhost,port=34799 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,898 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:33,899 DEBUG [Reader=1,bindAddress=localhost,port=34799 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,899 DEBUG [Reader=1,bindAddress=localhost,port=34799 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:33,899 INFO [Reader=1,bindAddress=localhost,port=34799 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:33,899 INFO [Reader=1,bindAddress=localhost,port=34799 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49452, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:33,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:33,905 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34799 2024-11-08T08:14:33,906 INFO [Listener,port=34799 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34799: stopping 2024-11-08T08:14:33,906 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:33,906 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:33,906 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:33,907 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41687 2024-11-08T08:14:33,907 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:33,907 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:33,908 INFO [Listener,port=41687 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41687: starting 2024-11-08T08:14:33,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d06cd9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:33,908 DEBUG [Thread-87 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:33,910 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,912 DEBUG [Reader=1,bindAddress=localhost,port=41687 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,913 DEBUG [Reader=1,bindAddress=localhost,port=41687 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:33,913 DEBUG [Reader=1,bindAddress=localhost,port=41687 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,914 DEBUG [Reader=1,bindAddress=localhost,port=41687 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:33,914 DEBUG [Reader=1,bindAddress=localhost,port=41687 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:33,914 DEBUG [Reader=1,bindAddress=localhost,port=41687 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,914 DEBUG [Reader=1,bindAddress=localhost,port=41687 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:33,915 DEBUG [Reader=1,bindAddress=localhost,port=41687 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:33,915 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:33,915 DEBUG [Reader=1,bindAddress=localhost,port=41687 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,915 DEBUG [Reader=1,bindAddress=localhost,port=41687 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,915 DEBUG [Reader=1,bindAddress=localhost,port=41687 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:33,915 INFO [Reader=1,bindAddress=localhost,port=41687 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:33,916 INFO [Reader=1,bindAddress=localhost,port=41687 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42578, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:33,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:33,921 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41687 2024-11-08T08:14:33,922 INFO [Listener,port=41687 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41687: stopping 2024-11-08T08:14:33,922 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:33,922 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:33,922 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:33,923 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=33585 2024-11-08T08:14:33,924 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:33,924 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:33,924 INFO [Listener,port=33585 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=33585: starting 2024-11-08T08:14:33,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ab0ad60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:33,925 DEBUG [Thread-90 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:33,926 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,929 DEBUG [Reader=1,bindAddress=localhost,port=33585 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,930 DEBUG [Reader=1,bindAddress=localhost,port=33585 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:33,930 DEBUG [Reader=1,bindAddress=localhost,port=33585 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,931 DEBUG [Reader=1,bindAddress=localhost,port=33585 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:33,932 DEBUG [Reader=1,bindAddress=localhost,port=33585 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:33,932 DEBUG [Reader=1,bindAddress=localhost,port=33585 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,932 DEBUG [Reader=1,bindAddress=localhost,port=33585 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:33,933 DEBUG [Reader=1,bindAddress=localhost,port=33585 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:33,933 DEBUG [Reader=1,bindAddress=localhost,port=33585 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,933 DEBUG [Reader=1,bindAddress=localhost,port=33585 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,933 DEBUG [Reader=1,bindAddress=localhost,port=33585 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int 2024-11-08T08:14:33,933 INFO [Reader=1,bindAddress=localhost,port=33585 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:33,933 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:33,935 INFO [Reader=1,bindAddress=localhost,port=33585 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60320, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:33,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:33,944 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 33585 2024-11-08T08:14:33,944 INFO [Listener,port=33585 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=33585: stopping 2024-11-08T08:14:33,944 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:33,945 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:33,945 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:33,946 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40509 2024-11-08T08:14:33,946 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:33,947 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:33,947 INFO [Listener,port=40509 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40509: starting 2024-11-08T08:14:33,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c88e50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:33,947 DEBUG [Thread-93 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:33,949 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,951 DEBUG [Reader=1,bindAddress=localhost,port=40509 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,951 DEBUG [Reader=1,bindAddress=localhost,port=40509 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:33,951 DEBUG [Reader=1,bindAddress=localhost,port=40509 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,952 DEBUG [Reader=1,bindAddress=localhost,port=40509 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:33,953 DEBUG [Reader=1,bindAddress=localhost,port=40509 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:33,953 DEBUG [Reader=1,bindAddress=localhost,port=40509 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,953 DEBUG [Reader=1,bindAddress=localhost,port=40509 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:33,953 DEBUG [Reader=1,bindAddress=localhost,port=40509 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:33,953 DEBUG [Reader=1,bindAddress=localhost,port=40509 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:33,953 DEBUG [Reader=1,bindAddress=localhost,port=40509 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:33,954 DEBUG [Reader=1,bindAddress=localhost,port=40509 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-08T08:14:33,954 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:33,954 INFO [Reader=1,bindAddress=localhost,port=40509 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:33,955 INFO [Reader=1,bindAddress=localhost,port=40509 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55054, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:33,962 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:33,963 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40509 2024-11-08T08:14:33,963 INFO [Listener,port=40509 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40509: stopping 2024-11-08T08:14:33,963 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:33,963 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:33,970 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=63 (was 57) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Reader=0,bindAddress=localhost,port=40509 java.base@17.0.11/java.lang.Thread.exit(Thread.java:864) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=886 (was 886), ProcessCount=11 (was 11), AvailableMemoryMB=6010 (was 6034) 2024-11-08T08:14:33,978 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=62, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=886, ProcessCount=11, AvailableMemoryMB=6009 2024-11-08T08:14:33,988 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:33,988 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:33,991 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053673990,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:33,993 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:33,994 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=42843 2024-11-08T08:14:33,997 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:33,997 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:33,997 INFO [Listener,port=42843 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=42843: starting 2024-11-08T08:14:33,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ce0d5dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:33,998 DEBUG [Thread-96 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-08T08:14:34,000 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(424): process preamble call response with response type null 2024-11-08T08:14:34,000 WARN [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call 2024-11-08T08:14:34,001 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes 2024-11-08T08:14:34,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:34,001 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 42843 2024-11-08T08:14:34,001 INFO [Listener,port=42843 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=42843: stopping 2024-11-08T08:14:34,001 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:34,002 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:34,007 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=63 (was 62) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=886 (was 886), ProcessCount=11 (was 11), AvailableMemoryMB=6009 (was 6009) 2024-11-08T08:14:34,011 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=63, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=886, ProcessCount=11, AvailableMemoryMB=6008 2024-11-08T08:14:34,020 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:34,020 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:34,023 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053674022,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:34,024 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:34,026 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=43773 2024-11-08T08:14:34,026 WARN [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! ********* 2024-11-08T08:14:34,026 WARN [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients 2024-11-08T08:14:34,026 WARN [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true). 2024-11-08T08:14:34,026 WARN [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user 2024-11-08T08:14:34,026 WARN [Time-limited test {}] ipc.RpcServer(380): impersonation is possible! 2024-11-08T08:14:34,026 WARN [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible, 2024-11-08T08:14:34,026 WARN [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml 2024-11-08T08:14:34,026 WARN [Time-limited test {}] ipc.RpcServer(383): **************************** 2024-11-08T08:14:34,027 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:34,027 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:34,027 INFO [Listener,port=43773 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=43773: starting 2024-11-08T08:14:34,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44b36892, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:34,028 DEBUG [Thread-99 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-08T08:14:34,031 WARN [Reader=1,bindAddress=localhost,port=43773 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1 2024-11-08T08:14:34,031 INFO [Reader=1,bindAddress=localhost,port=43773 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-08T08:14:34,037 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:34,037 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 43773 2024-11-08T08:14:34,037 INFO [Listener,port=43773 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=43773: stopping 2024-11-08T08:14:34,038 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:34,039 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:34,045 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=66 (was 63) Potentially hanging thread: FifoRpcScheduler.handler-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.getAndBitwiseAndInt(Unsafe.java:3227) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$Node.getAndUnsetStatus(AbstractQueuedSynchronizer.java:468) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1618) java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) java.base@17.0.11/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647) app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$15(StackLocator.java:100) app//org.apache.logging.log4j.util.StackLocator$$Lambda$125/0x00007f2d2418c260.apply(Unknown Source) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243) java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499) app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97) app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121) app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=886 (was 886), ProcessCount=11 (was 11), AvailableMemoryMB=6006 (was 6008) 2024-11-08T08:14:34,053 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=64, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=886, ProcessCount=11, AvailableMemoryMB=6006 2024-11-08T08:14:34,062 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:34,062 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:34,064 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053674064,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:34,066 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:34,067 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40069 2024-11-08T08:14:34,068 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:34,068 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:34,068 INFO [Listener,port=40069 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40069: starting 2024-11-08T08:14:34,068 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5814822d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:34,069 DEBUG [Thread-102 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:34,071 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:34,074 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053674074,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:34,076 DEBUG [Reader=1,bindAddress=localhost,port=40069 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:34,077 DEBUG [Reader=1,bindAddress=localhost,port=40069 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:34,077 DEBUG [Reader=1,bindAddress=localhost,port=40069 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:34,078 DEBUG [Reader=1,bindAddress=localhost,port=40069 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:34,078 DEBUG [Reader=1,bindAddress=localhost,port=40069 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:34,078 DEBUG [Reader=1,bindAddress=localhost,port=40069 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:34,079 DEBUG [Reader=1,bindAddress=localhost,port=40069 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:34,079 DEBUG [Reader=1,bindAddress=localhost,port=40069 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:34,079 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:34,079 DEBUG [Reader=1,bindAddress=localhost,port=40069 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:34,079 DEBUG [Reader=1,bindAddress=localhost,port=40069 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:34,080 DEBUG [Reader=1,bindAddress=localhost,port=40069 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:34,080 INFO [Reader=1,bindAddress=localhost,port=40069 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:34,080 INFO [Reader=1,bindAddress=localhost,port=40069 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50948, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:34,090 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:34,090 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40069 2024-11-08T08:14:34,090 INFO [Listener,port=40069 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40069: stopping 2024-11-08T08:14:34,091 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:34,091 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:34,098 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=68 (was 64) Potentially hanging thread: Reader=1,bindAddress=localhost,port=40069 java.base@17.0.11/java.lang.ThreadLocal.isPresent(ThreadLocal.java:184) java.base@17.0.11/java.lang.Thread.exit(Thread.java:849) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: FifoRpcScheduler.handler-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer idle connection scanner for port 40069 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=886 (was 886), ProcessCount=11 (was 11), AvailableMemoryMB=6004 (was 6006) 2024-11-08T08:14:34,108 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=65, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=886, ProcessCount=11, AvailableMemoryMB=6004 2024-11-08T08:14:34,117 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:34,118 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:34,119 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053674119,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:34,121 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:34,122 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=36547 2024-11-08T08:14:34,124 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:34,128 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:34,128 INFO [Listener,port=36547 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=36547: starting 2024-11-08T08:14:34,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@992054d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:34,130 DEBUG [Thread-105 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:34,133 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:34,137 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053674136,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:34,139 DEBUG [Reader=1,bindAddress=localhost,port=36547 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:34,140 DEBUG [Reader=1,bindAddress=localhost,port=36547 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:34,140 DEBUG [Reader=1,bindAddress=localhost,port=36547 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:34,141 DEBUG [Reader=1,bindAddress=localhost,port=36547 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:34,142 DEBUG [Reader=1,bindAddress=localhost,port=36547 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:34,142 DEBUG [Reader=1,bindAddress=localhost,port=36547 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:34,142 DEBUG [Reader=1,bindAddress=localhost,port=36547 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:34,143 DEBUG [Reader=1,bindAddress=localhost,port=36547 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:34,144 DEBUG [Reader=1,bindAddress=localhost,port=36547 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:34,144 DEBUG [Reader=1,bindAddress=localhost,port=36547 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:34,144 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:34,144 DEBUG [Reader=1,bindAddress=localhost,port=36547 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-08T08:14:34,144 INFO [Reader=1,bindAddress=localhost,port=36547 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:34,145 INFO [Reader=1,bindAddress=localhost,port=36547 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56530, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:34,167 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:34,167 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 36547 2024-11-08T08:14:34,167 INFO [Listener,port=36547 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=36547: stopping 2024-11-08T08:14:34,167 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:34,167 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:34,168 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:34,192 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=38965 2024-11-08T08:14:34,208 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:34,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32cdc312, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:34,211 INFO [Listener,port=38965 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=38965: starting 2024-11-08T08:14:34,211 DEBUG [Thread-108 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:34,216 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:34,218 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:34,221 DEBUG [Reader=1,bindAddress=localhost,port=38965 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:34,222 DEBUG [Reader=1,bindAddress=localhost,port=38965 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:34,222 DEBUG [Reader=1,bindAddress=localhost,port=38965 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:34,223 DEBUG [Reader=1,bindAddress=localhost,port=38965 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:34,224 DEBUG [Reader=1,bindAddress=localhost,port=38965 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:34,224 DEBUG [Reader=1,bindAddress=localhost,port=38965 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:34,224 DEBUG [Reader=1,bindAddress=localhost,port=38965 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:34,225 DEBUG [Reader=1,bindAddress=localhost,port=38965 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:34,225 DEBUG [Reader=1,bindAddress=localhost,port=38965 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:34,226 DEBUG [Reader=1,bindAddress=localhost,port=38965 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:34,226 DEBUG [Reader=1,bindAddress=localhost,port=38965 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-08T08:14:34,226 INFO [Reader=1,bindAddress=localhost,port=38965 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:34,227 INFO [Reader=1,bindAddress=localhost,port=38965 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40114, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:36,229 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-08T08:14:36,230 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:36,230 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 38965 2024-11-08T08:14:36,230 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.FailedServers(52): Added failed server with address localhost:38965 to list caused by java.io.IOException: org.apache.hbase.thirdparty.io.netty.handler.timeout.ReadTimeoutException 2024-11-08T08:14:36,230 INFO [Listener,port=38965 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=38965: stopping 2024-11-08T08:14:36,230 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:36,230 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:36,240 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=68 (was 65) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Reader=1,bindAddress=localhost,port=38965 java.base@17.0.11/java.nio.channels.spi.AbstractSelectableChannel.isRegistered(AbstractSelectableChannel.java:165) java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:193) java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:151) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=886 (was 886), ProcessCount=11 (was 11), AvailableMemoryMB=6621 (was 6004) - AvailableMemoryMB LEAK? - 2024-11-08T08:14:36,249 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=67, OpenFileDescriptor=468, MaxFileDescriptor=1048576, SystemLoadAverage=886, ProcessCount=11, AvailableMemoryMB=6617 2024-11-08T08:14:36,265 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:36,265 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:36,269 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053676268,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:36,274 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:36,281 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39415 2024-11-08T08:14:36,284 WARN [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731053676273 2024-11-08T08:14:36,288 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:36,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55208a25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:36,290 INFO [Listener,port=39415 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39415: starting 2024-11-08T08:14:36,292 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:36,297 DEBUG [Thread-111 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:36,302 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,308 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053676307,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,313 DEBUG [Reader=1,bindAddress=localhost,port=39415 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,314 DEBUG [Reader=1,bindAddress=localhost,port=39415 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:36,314 DEBUG [Reader=1,bindAddress=localhost,port=39415 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,315 DEBUG [Reader=1,bindAddress=localhost,port=39415 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:36,320 DEBUG [Reader=1,bindAddress=localhost,port=39415 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:36,320 DEBUG [Reader=1,bindAddress=localhost,port=39415 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,320 DEBUG [Reader=1,bindAddress=localhost,port=39415 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:36,323 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:36,328 DEBUG [Reader=1,bindAddress=localhost,port=39415 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:36,328 DEBUG [Reader=1,bindAddress=localhost,port=39415 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,328 DEBUG [Reader=1,bindAddress=localhost,port=39415 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,329 DEBUG [Reader=1,bindAddress=localhost,port=39415 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:36,329 INFO [Reader=1,bindAddress=localhost,port=39415 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:36,329 INFO [Reader=1,bindAddress=localhost,port=39415 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49284, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:36,341 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:36,341 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39415 2024-11-08T08:14:36,344 INFO [Listener,port=39415 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39415: stopping 2024-11-08T08:14:36,344 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:36,348 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:36,351 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=72 (was 67) Potentially hanging thread: RpcServer idle connection scanner for port 39415 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Listener,port=39415 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:213) Potentially hanging thread: Reader=0,bindAddress=localhost,port=39415 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer.responder java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:115) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=468 (was 468), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=886 (was 886), ProcessCount=11 (was 11), AvailableMemoryMB=6614 (was 6617) 2024-11-08T08:14:36,364 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=68, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=886, ProcessCount=11, AvailableMemoryMB=6614 2024-11-08T08:14:36,378 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:36,378 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:36,381 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053676381,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:36,384 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:36,399 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=42599 2024-11-08T08:14:36,407 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:36,407 INFO [Listener,port=42599 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=42599: starting 2024-11-08T08:14:36,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@167460ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:36,408 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:36,409 DEBUG [Thread-114 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:36,412 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,415 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053676415,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,417 DEBUG [Reader=1,bindAddress=localhost,port=42599 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,417 DEBUG [Reader=1,bindAddress=localhost,port=42599 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:36,418 DEBUG [Reader=1,bindAddress=localhost,port=42599 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,418 DEBUG [Reader=1,bindAddress=localhost,port=42599 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-08T08:14:36,420 DEBUG [Reader=1,bindAddress=localhost,port=42599 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:36,420 DEBUG [Reader=1,bindAddress=localhost,port=42599 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,420 DEBUG [Reader=1,bindAddress=localhost,port=42599 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-08T08:14:36,421 DEBUG [Reader=1,bindAddress=localhost,port=42599 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-08T08:14:36,421 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:36,421 DEBUG [Reader=1,bindAddress=localhost,port=42599 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,421 DEBUG [Reader=1,bindAddress=localhost,port=42599 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,421 DEBUG [Reader=1,bindAddress=localhost,port=42599 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:36,421 INFO [Reader=1,bindAddress=localhost,port=42599 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:36,422 INFO [Reader=1,bindAddress=localhost,port=42599 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50394, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:36,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:36,431 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 42599 2024-11-08T08:14:36,431 INFO [Listener,port=42599 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=42599: stopping 2024-11-08T08:14:36,432 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:36,432 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:36,439 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=71 (was 68) Potentially hanging thread: Reader=0,bindAddress=localhost,port=42599 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer idle connection scanner for port 42599 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=468 (was 466) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=886 (was 886), ProcessCount=11 (was 11), AvailableMemoryMB=6615 (was 6614) - AvailableMemoryMB LEAK? - 2024-11-08T08:14:36,459 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=69, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=886, ProcessCount=11, AvailableMemoryMB=6607 2024-11-08T08:14:36,472 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:36,472 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:36,474 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053676474,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:36,477 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:36,481 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=36833 2024-11-08T08:14:36,482 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-08T08:14:36,482 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:36,482 INFO [Listener,port=36833 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=36833: starting 2024-11-08T08:14:36,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ce2a238, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:36,483 DEBUG [Thread-117 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:36,485 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,488 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053676488,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,490 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:36,491 INFO [Reader=1,bindAddress=localhost,port=36833 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-08T08:14:36,497 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:36,498 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 36833 2024-11-08T08:14:36,498 INFO [Listener,port=36833 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=36833: stopping 2024-11-08T08:14:36,498 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-08T08:14:36,498 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-08T08:14:36,506 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=70 (was 69) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=903 (was 886) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6605 (was 6607) 2024-11-08T08:14:36,514 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=70, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=903, ProcessCount=11, AvailableMemoryMB=6604 2024-11-08T08:14:36,524 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:36,525 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:36,526 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053676526,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:36,528 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:36,528 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:36,530 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39707 2024-11-08T08:14:36,530 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:36,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f87b856, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:36,531 DEBUG [Thread-118 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:36,532 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,537 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053676537,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,539 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,539 DEBUG [NettyRpcServer-19-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,541 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:36,542 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,542 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:36,542 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,543 DEBUG [NettyRpcServer-19-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,543 DEBUG [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-08T08:14:36,543 INFO [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:36,544 INFO [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34688, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:36,544 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:36,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:36,559 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39707 2024-11-08T08:14:36,571 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=73 (was 70) Potentially hanging thread: NettyRpcServer-19-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-19-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: FifoRpcScheduler.handler-pool-0 java.base@17.0.11/java.lang.Thread.currentThread(Native Method) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:638) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1636) java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=475 (was 466) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=903 (was 903), ProcessCount=11 (was 11), AvailableMemoryMB=6603 (was 6604) 2024-11-08T08:14:36,584 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=72, OpenFileDescriptor=475, MaxFileDescriptor=1048576, SystemLoadAverage=903, ProcessCount=11, AvailableMemoryMB=6603 2024-11-08T08:14:36,597 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:36,597 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:36,600 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053676600,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:36,603 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:36,603 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:36,604 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34437 2024-11-08T08:14:36,604 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:36,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c8102d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-08T08:14:36,609 DEBUG [Thread-119 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:36,610 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,613 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053676613,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,615 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-08T08:14:36,616 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.FailedServers(52): Added failed server with address localhost:34437 to list caused by org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections. 2024-11-08T08:14:36,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:36,616 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34437 2024-11-08T08:14:36,625 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=74 (was 72) Potentially hanging thread: NettyRpcServer-20-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-20-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=484 (was 475) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=903 (was 903), ProcessCount=11 (was 11), AvailableMemoryMB=6600 (was 6603) 2024-11-08T08:14:36,633 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=74, OpenFileDescriptor=484, MaxFileDescriptor=1048576, SystemLoadAverage=903, ProcessCount=11, AvailableMemoryMB=6600 2024-11-08T08:14:36,653 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:36,653 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:36,655 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053676655,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:36,657 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:36,658 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:36,664 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36931 2024-11-08T08:14:36,664 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:36,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13a2b806, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:36,671 DEBUG [Thread-120 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:36,673 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,677 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053676677,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,679 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,680 DEBUG [NettyRpcServer-21-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,682 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:36,684 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,684 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:36,685 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-08T08:14:36,685 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.FailedServers(52): Added failed server with address localhost:36931 to list caused by javax.security.sasl.SaslException: No common protection layer between client and server 2024-11-08T08:14:36,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:36,686 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36931 2024-11-08T08:14:36,698 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=76 (was 74) Potentially hanging thread: NettyRpcServer-21-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-21-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=493 (was 484) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=903 (was 903), ProcessCount=11 (was 11), AvailableMemoryMB=6598 (was 6600) 2024-11-08T08:14:36,709 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=76, OpenFileDescriptor=493, MaxFileDescriptor=1048576, SystemLoadAverage=903, ProcessCount=11, AvailableMemoryMB=6596 2024-11-08T08:14:36,726 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:36,726 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:36,729 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053676729,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:36,735 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:36,735 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:36,736 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38507 2024-11-08T08:14:36,736 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:36,736 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e2e969, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:36,745 DEBUG [Thread-121 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:36,746 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,750 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053676749,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,753 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,753 DEBUG [NettyRpcServer-22-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,755 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:36,760 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,760 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:36,761 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:36,764 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,764 DEBUG [NettyRpcServer-22-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,765 DEBUG [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:36,765 INFO [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:36,765 INFO [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43832, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:36,768 WARN [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731053676733 2024-11-08T08:14:36,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:36,793 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38507 2024-11-08T08:14:36,793 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:36,794 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:36,796 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35223 2024-11-08T08:14:36,796 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:36,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64e5a988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:36,852 DEBUG [Thread-122 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:36,854 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,855 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,855 DEBUG [NettyRpcServer-23-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,857 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:36,858 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,858 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:36,859 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,859 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:36,859 DEBUG [NettyRpcServer-23-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,859 DEBUG [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:36,859 INFO [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:36,860 INFO [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57948, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:36,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:36,875 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35223 2024-11-08T08:14:36,875 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:36,876 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:36,877 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36593 2024-11-08T08:14:36,877 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:36,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@227ace28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:36,878 DEBUG [Thread-123 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:36,879 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,885 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,885 DEBUG [NettyRpcServer-24-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,887 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:36,891 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,892 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:36,892 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:36,900 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,900 DEBUG [NettyRpcServer-24-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,901 DEBUG [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:36,901 INFO [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:36,905 INFO [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41284, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:36,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:36,942 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36593 2024-11-08T08:14:36,947 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:36,947 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:36,956 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43879 2024-11-08T08:14:36,957 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:36,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@403cc143, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:36,961 DEBUG [Thread-124 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:36,965 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,973 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,973 DEBUG [NettyRpcServer-25-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,975 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:36,977 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,977 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:36,978 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:36,978 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:36,979 DEBUG [NettyRpcServer-25-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:36,979 DEBUG [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int 2024-11-08T08:14:36,979 INFO [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:36,980 INFO [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38872, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:37,004 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:37,004 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43879 2024-11-08T08:14:37,005 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:37,008 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:37,012 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35469 2024-11-08T08:14:37,013 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:37,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6936ad7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:37,017 DEBUG [Thread-125 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:37,018 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:37,021 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=498 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:37,021 DEBUG [NettyRpcServer-26-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:37,023 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 107 from saslServer. 2024-11-08T08:14:37,024 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:37,024 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:37,024 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:37,025 DEBUG [NettyRpcServer-26-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:37,025 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:37,025 DEBUG [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-08T08:14:37,025 INFO [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:37,026 INFO [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36702, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:37,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:37,044 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35469 2024-11-08T08:14:37,067 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=86 (was 76) Potentially hanging thread: NettyRpcServer-23-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-22-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-25-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-24-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-26-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-23-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-25-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-22-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-24-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-26-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=538 (was 493) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=903 (was 903), ProcessCount=11 (was 11), AvailableMemoryMB=6559 (was 6596) 2024-11-08T08:14:37,079 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=86, OpenFileDescriptor=538, MaxFileDescriptor=1048576, SystemLoadAverage=903, ProcessCount=11, AvailableMemoryMB=6558 2024-11-08T08:14:37,093 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:37,093 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:37,096 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053677095,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:37,099 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:37,099 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:37,104 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36489 2024-11-08T08:14:37,105 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:37,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c886caf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:37,109 DEBUG [Thread-126 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-08T08:14:37,112 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(424): process preamble call response with response type null 2024-11-08T08:14:37,112 WARN [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call 2024-11-08T08:14:37,112 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes 2024-11-08T08:14:37,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:37,113 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36489 2024-11-08T08:14:37,126 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=88 (was 86) Potentially hanging thread: NettyRpcServer-27-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-27-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=547 (was 538) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=903 (was 903), ProcessCount=11 (was 11), AvailableMemoryMB=6552 (was 6558) 2024-11-08T08:14:37,138 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=88, OpenFileDescriptor=547, MaxFileDescriptor=1048576, SystemLoadAverage=903, ProcessCount=11, AvailableMemoryMB=6551 2024-11-08T08:14:37,151 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:37,152 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:37,154 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053677154,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:37,156 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:37,156 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:37,157 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43055 2024-11-08T08:14:37,158 WARN [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! ********* 2024-11-08T08:14:37,158 WARN [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients 2024-11-08T08:14:37,158 WARN [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true). 2024-11-08T08:14:37,158 WARN [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user 2024-11-08T08:14:37,158 WARN [Time-limited test {}] ipc.RpcServer(380): impersonation is possible! 2024-11-08T08:14:37,158 WARN [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible, 2024-11-08T08:14:37,158 WARN [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml 2024-11-08T08:14:37,158 WARN [Time-limited test {}] ipc.RpcServer(383): **************************** 2024-11-08T08:14:37,158 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:37,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55c9b3f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:37,163 DEBUG [Thread-127 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-08T08:14:37,165 WARN [NettyRpcServer-28-2 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1 2024-11-08T08:14:37,165 INFO [NettyRpcServer-28-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-08T08:14:37,174 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:37,174 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43055 2024-11-08T08:14:37,182 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=90 (was 88) Potentially hanging thread: NettyRpcServer-28-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-28-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=556 (was 547) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=903 (was 903), ProcessCount=11 (was 11), AvailableMemoryMB=6548 (was 6551) 2024-11-08T08:14:37,192 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=90, OpenFileDescriptor=556, MaxFileDescriptor=1048576, SystemLoadAverage=903, ProcessCount=11, AvailableMemoryMB=6547 2024-11-08T08:14:37,205 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:37,205 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:37,208 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053677207,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:37,211 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:37,211 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:37,217 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35681 2024-11-08T08:14:37,217 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:37,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e5b59af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:37,219 DEBUG [Thread-128 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:37,221 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:37,223 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053677223,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:37,229 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:37,229 DEBUG [NettyRpcServer-29-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:37,231 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:37,234 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:37,234 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:37,235 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:37,235 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:37,235 DEBUG [NettyRpcServer-29-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:37,235 DEBUG [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:37,235 INFO [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:37,236 INFO [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38802, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:37,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:37,256 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35681 2024-11-08T08:14:37,269 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=92 (was 90) Potentially hanging thread: NettyRpcServer-29-1 app//org.apache.hbase.thirdparty.io.netty.channel.unix.FileDescriptor.close(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.unix.FileDescriptor.close(FileDescriptor.java:83) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel.doClose(AbstractEpollChannel.java:191) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:752) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:729) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:619) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.close(DefaultChannelPipeline.java:1299) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeClose(AbstractChannelHandlerContext.java:755) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.access$1200(AbstractChannelHandlerContext.java:61) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext$11.run(AbstractChannelHandlerContext.java:738) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:405) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-29-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=565 (was 556) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=903 (was 903), ProcessCount=11 (was 11), AvailableMemoryMB=6540 (was 6547) 2024-11-08T08:14:37,278 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=92, OpenFileDescriptor=565, MaxFileDescriptor=1048576, SystemLoadAverage=903, ProcessCount=11, AvailableMemoryMB=6539 2024-11-08T08:14:37,293 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:37,293 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:37,295 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053677295,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:37,297 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:37,297 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:37,303 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40321 2024-11-08T08:14:37,303 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:37,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5613266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:37,305 DEBUG [Thread-129 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:37,306 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:37,309 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053677309,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:37,310 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:37,311 DEBUG [NettyRpcServer-30-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:37,312 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:37,313 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:37,313 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:37,314 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:37,314 DEBUG [NettyRpcServer-30-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:37,314 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:37,314 DEBUG [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-08T08:14:37,314 INFO [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:37,315 INFO [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33372, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:37,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:37,327 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40321 2024-11-08T08:14:37,327 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:37,328 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:37,329 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45463 2024-11-08T08:14:37,329 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:37,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3049951b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:37,332 DEBUG [Thread-130 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:37,334 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:37,338 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:37,338 DEBUG [NettyRpcServer-31-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:37,339 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:37,340 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:37,340 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:37,341 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:37,341 DEBUG [NettyRpcServer-31-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:37,341 DEBUG [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-08T08:14:37,341 INFO [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:37,342 INFO [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35116, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:39,343 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-08T08:14:39,344 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:39,344 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.FailedServers(52): Added failed server with address localhost:45463 to list caused by java.io.IOException: org.apache.hbase.thirdparty.io.netty.handler.timeout.ReadTimeoutException 2024-11-08T08:14:39,344 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45463 2024-11-08T08:14:39,354 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=96 (was 92) Potentially hanging thread: NettyRpcServer-31-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-31-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-30-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-30-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=583 (was 565) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=903 (was 903), ProcessCount=11 (was 11), AvailableMemoryMB=6419 (was 6539) 2024-11-08T08:14:39,364 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=96, OpenFileDescriptor=583, MaxFileDescriptor=1048576, SystemLoadAverage=903, ProcessCount=11, AvailableMemoryMB=6417 2024-11-08T08:14:39,368 WARN [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731053677296 2024-11-08T08:14:39,397 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:39,397 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:39,404 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053679403,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:39,406 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:39,407 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:39,408 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41833 2024-11-08T08:14:39,408 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:39,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26d9b868, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:39,409 DEBUG [Thread-131 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:39,413 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:39,416 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053679416,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:39,429 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:39,429 DEBUG [NettyRpcServer-32-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:39,431 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:39,433 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:39,433 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:39,434 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:39,435 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:39,435 DEBUG [NettyRpcServer-32-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:39,435 DEBUG [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:39,435 INFO [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:39,436 INFO [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38542, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:39,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:39,448 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41833 2024-11-08T08:14:39,463 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=99 (was 96) Potentially hanging thread: NettyRpcServer-32-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-32-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: FifoRpcScheduler.handler-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=592 (was 583) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=903 (was 903), ProcessCount=11 (was 11), AvailableMemoryMB=6413 (was 6417) 2024-11-08T08:14:39,476 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=98, OpenFileDescriptor=592, MaxFileDescriptor=1048576, SystemLoadAverage=903, ProcessCount=11, AvailableMemoryMB=6413 2024-11-08T08:14:39,487 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:39,488 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:39,490 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053679489,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:39,491 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:39,492 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:39,497 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33431 2024-11-08T08:14:39,497 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:39,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16457b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:39,500 DEBUG [Thread-132 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:39,501 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:39,504 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053679504,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:39,506 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:39,506 DEBUG [NettyRpcServer-33-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:39,508 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-08T08:14:39,508 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:39,509 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-08T08:14:39,509 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-08T08:14:39,509 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:39,509 DEBUG [NettyRpcServer-33-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:39,509 DEBUG [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-08T08:14:39,510 INFO [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-08T08:14:39,510 INFO [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35940, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-08T08:14:39,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:39,518 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33431 2024-11-08T08:14:39,526 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=100 (was 98) Potentially hanging thread: NettyRpcServer-33-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-33-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=601 (was 592) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=903 (was 903), ProcessCount=11 (was 11), AvailableMemoryMB=6410 (was 6413) 2024-11-08T08:14:39,534 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=100, OpenFileDescriptor=601, MaxFileDescriptor=1048576, SystemLoadAverage=903, ProcessCount=11, AvailableMemoryMB=6410 2024-11-08T08:14:39,544 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-08T08:14:39,545 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-08T08:14:39,547 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731053679546,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-08T08:14:39,549 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-08T08:14:39,549 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T08:14:39,551 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:32935 2024-11-08T08:14:39,551 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-08T08:14:39,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24b5e489, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T08:14:39,552 DEBUG [Thread-133 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-08T08:14:39,553 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:39,556 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731053679556,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-08T08:14:39,561 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-08T08:14:39,562 INFO [NettyRpcServer-34-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-08T08:14:39,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T08:14:39,574 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:32935 2024-11-08T08:14:39,585 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=102 (was 100) Potentially hanging thread: NettyRpcServer-34-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-34-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=610 (was 601) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=903 (was 903), ProcessCount=11 (was 11), AvailableMemoryMB=6404 (was 6410) 2024-11-08T08:14:39,586 INFO [Time-limited test {}] impl.DefaultInternalKdcServerImpl(102): Default Internal kdc server stopped.