2024-11-16 14:39:47,612 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 2024-11-16 14:39:47,630 main DEBUG Took 0.014786 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 14:39:47,631 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 14:39:47,632 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 14:39:47,633 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 14:39:47,635 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:39:47,656 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 14:39:47,678 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:39:47,680 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:39:47,681 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:39:47,682 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:39:47,683 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:39:47,683 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:39:47,684 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:39:47,685 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:39:47,686 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:39:47,686 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:39:47,688 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:39:47,688 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:39:47,689 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:39:47,689 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:39:47,690 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:39:47,691 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:39:47,691 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:39:47,692 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:39:47,692 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:39:47,693 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:39:47,694 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:39:47,694 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:39:47,695 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:39:47,695 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:39:47,696 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:39:47,696 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 14:39:47,698 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:39:47,700 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 14:39:47,703 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 14:39:47,703 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 14:39:47,705 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 14:39:47,706 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 14:39:47,717 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 14:39:47,721 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 14:39:47,724 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 14:39:47,724 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 14:39:47,725 main DEBUG createAppenders(={Console}) 2024-11-16 14:39:47,726 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 initialized 2024-11-16 14:39:47,727 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 2024-11-16 14:39:47,727 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 OK. 2024-11-16 14:39:47,728 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 14:39:47,729 main DEBUG OutputStream closed 2024-11-16 14:39:47,729 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 14:39:47,729 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 14:39:47,730 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5d0a1059 OK 2024-11-16 14:39:47,837 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 14:39:47,840 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 14:39:47,841 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 14:39:47,842 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 14:39:47,843 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 14:39:47,843 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 14:39:47,844 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 14:39:47,844 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 14:39:47,845 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 14:39:47,845 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 14:39:47,846 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 14:39:47,846 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 14:39:47,846 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 14:39:47,847 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 14:39:47,847 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 14:39:47,847 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 14:39:47,848 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 14:39:47,849 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 14:39:47,851 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 14:39:47,852 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5812f68b) with optional ClassLoader: null 2024-11-16 14:39:47,852 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 14:39:47,853 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5812f68b] started OK. 2024-11-16T14:39:48,237 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d8a09a53-9f7e-0481-74e3-e1745f2c1f1f 2024-11-16 14:39:48,241 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 14:39:48,241 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T14:39:48,259 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.security.TestSecureIPC timeout: 52 mins 2024-11-16T14:39:48,531 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=13, OpenFileDescriptor=288, MaxFileDescriptor=1048576, SystemLoadAverage=315, ProcessCount=11, AvailableMemoryMB=7500 2024-11-16T14:39:48,622 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T14:39:48,692 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:48,699 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:48,765 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767988762,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:49,496 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:49,593 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T14:39:49,635 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T14:39:49,649 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16T14:39:49,672 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=35077 2024-11-16T14:39:49,680 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:49,680 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:49,680 INFO [Listener,port=35077 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=35077: starting 2024-11-16T14:39:49,739 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de99998, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:49,795 DEBUG [Thread-6 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:49,853 DEBUG [Thread-6 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:35077 2024-11-16T14:39:49,864 DEBUG [Thread-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:49,915 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767989913,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:49,924 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:49,935 DEBUG [Reader=1,bindAddress=localhost,port=35077 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:49,940 DEBUG [Reader=1,bindAddress=localhost,port=35077 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:49,940 DEBUG [Reader=1,bindAddress=localhost,port=35077 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:49,947 DEBUG [Reader=1,bindAddress=localhost,port=35077 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:49,949 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:49,951 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:49,951 DEBUG [Reader=1,bindAddress=localhost,port=35077 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:49,952 DEBUG [Reader=1,bindAddress=localhost,port=35077 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:49,953 DEBUG [Reader=1,bindAddress=localhost,port=35077 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:49,953 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:49,954 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:49,955 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-16T14:39:49,955 DEBUG [Reader=1,bindAddress=localhost,port=35077 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:49,955 DEBUG [Reader=1,bindAddress=localhost,port=35077 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:49,956 DEBUG [Thread-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:49,956 DEBUG [Reader=1,bindAddress=localhost,port=35077 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:49,956 DEBUG [Reader=1,bindAddress=localhost,port=35077 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-16T14:39:49,957 INFO [Reader=1,bindAddress=localhost,port=35077 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:49,977 INFO [Reader=1,bindAddress=localhost,port=35077 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38146, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:49,977 DEBUG [Thread-6 {}] ipc.BlockingRpcConnection(634): Length of response for connection header:93 2024-11-16T14:39:49,991 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:246 2024-11-16T14:39:49,993 DEBUG [BRPC Connection (1845770469) to localhost:35077 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-16T14:39:50,022 DEBUG [BRPC Connection (1845770469) to localhost:35077 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:232 2024-11-16T14:39:50,027 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:2278 2024-11-16T14:39:50,028 DEBUG [BRPC Connection (1845770469) to localhost:35077 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-16T14:39:50,032 DEBUG [BRPC Connection (1845770469) to localhost:35077 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:2264 2024-11-16T14:39:50,047 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:4117 2024-11-16T14:39:50,048 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:18257 2024-11-16T14:39:50,052 DEBUG [BRPC Connection (1845770469) to localhost:35077 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-16T14:39:50,070 DEBUG [BRPC Connection (1845770469) to localhost:35077 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:22360 2024-11-16T14:39:50,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:50,080 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 35077 2024-11-16T14:39:50,080 INFO [Listener,port=35077 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=35077: stopping 2024-11-16T14:39:50,081 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:50,081 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:50,090 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 13) Potentially hanging thread: Reader=1,bindAddress=localhost,port=35077 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: pool-3-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=292 (was 288) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 315) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7297 (was 7500) 2024-11-16T14:39:50,096 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=7296 2024-11-16T14:39:50,109 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:50,110 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:50,118 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767990117,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:50,137 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:50,139 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=46273 2024-11-16T14:39:50,140 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:50,140 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:50,140 INFO [Listener,port=46273 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=46273: starting 2024-11-16T14:39:50,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54cfc19d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-16T14:39:50,142 DEBUG [Thread-11 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:50,143 DEBUG [Thread-11 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:46273 2024-11-16T14:39:50,144 DEBUG [Thread-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,154 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767990154,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,158 DEBUG [Thread-11 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:50,159 DEBUG [Thread-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-16T14:39:50,160 DEBUG [Thread-11 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:46273 org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections. at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:111) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-16T14:39:50,165 WARN [Thread-11 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731767990121 2024-11-16T14:39:50,185 DEBUG [Thread-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,188 DEBUG [Thread-11 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:50,189 DEBUG [Thread-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-16T14:39:50,190 DEBUG [Thread-11 {}] ipc.FailedServers(52): Added failed server with address localhost:46273 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:50,191 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 46273 2024-11-16T14:39:50,191 INFO [Listener,port=46273 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=46273: stopping 2024-11-16T14:39:50,192 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:50,192 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:50,199 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=24 (was 19) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.nio.ByteBuffer.clear(ByteBuffer.java:267) app//org.apache.logging.log4j.core.layout.TextEncoderHelper.writeEncodedText(TextEncoderHelper.java:97) app//org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeText(TextEncoderHelper.java:65) app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:68) app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:32) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:227) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) Potentially hanging thread: Reader=2,bindAddress=localhost,port=46273 java.base@17.0.11/java.lang.ThreadGroup.threadTerminated(ThreadGroup.java:967) java.base@17.0.11/java.lang.Thread.exit(Thread.java:853) Potentially hanging thread: Reader=1,bindAddress=localhost,port=46273 java.base@17.0.11/java.lang.ThreadGroup.threadTerminated(ThreadGroup.java:967) java.base@17.0.11/java.lang.Thread.exit(Thread.java:853) Potentially hanging thread: RpcServer idle connection scanner for port 46273 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Listener,port=46273 java.base@17.0.11/java.lang.System.arraycopy(Native Method) java.base@17.0.11/java.lang.ThreadGroup.remove(ThreadGroup.java:994) java.base@17.0.11/java.lang.ThreadGroup.threadTerminated(ThreadGroup.java:967) java.base@17.0.11/java.lang.Thread.exit(Thread.java:853) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 330), ProcessCount=11 (was 11), AvailableMemoryMB=7285 (was 7296) 2024-11-16T14:39:50,207 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=7282 2024-11-16T14:39:50,224 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:50,224 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:50,232 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767990231,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:50,236 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:50,238 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=38741 2024-11-16T14:39:50,239 INFO [Listener,port=38741 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=38741: starting 2024-11-16T14:39:50,239 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:50,239 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:50,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ae19adf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:50,241 DEBUG [Thread-14 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:50,241 DEBUG [Thread-14 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:38741 2024-11-16T14:39:50,242 DEBUG [Thread-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,250 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767990250,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,253 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:50,253 DEBUG [Reader=1,bindAddress=localhost,port=38741 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,255 DEBUG [Reader=1,bindAddress=localhost,port=38741 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,255 DEBUG [Reader=1,bindAddress=localhost,port=38741 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,257 DEBUG [Reader=1,bindAddress=localhost,port=38741 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:50,257 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:50,258 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:50,258 DEBUG [Reader=1,bindAddress=localhost,port=38741 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,259 DEBUG [Reader=1,bindAddress=localhost,port=38741 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,259 DEBUG [Reader=1,bindAddress=localhost,port=38741 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:50,260 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:50,260 DEBUG [Thread-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-16T14:39:50,260 DEBUG [Thread-14 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:38741 javax.security.sasl.SaslException: No common protection layer between client and server at com.sun.security.sasl.gsskerb.GssKrb5Client.doFinalHandshake(GssKrb5Client.java:270) ~[jdk.security.jgss:?] at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:203) ~[jdk.security.jgss:?] at org.apache.hadoop.hbase.security.AbstractHBaseSaslRpcClient.evaluateChallenge(AbstractHBaseSaslRpcClient.java:105) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:126) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-16T14:39:50,261 WARN [Thread-14 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731767990234 2024-11-16T14:39:50,273 DEBUG [Thread-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,275 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:50,275 DEBUG [Reader=2,bindAddress=localhost,port=38741 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,277 DEBUG [Reader=2,bindAddress=localhost,port=38741 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,277 DEBUG [Reader=2,bindAddress=localhost,port=38741 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,280 DEBUG [Reader=2,bindAddress=localhost,port=38741 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:50,280 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:50,281 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:50,281 DEBUG [Reader=2,bindAddress=localhost,port=38741 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,282 DEBUG [Reader=2,bindAddress=localhost,port=38741 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,282 DEBUG [Reader=2,bindAddress=localhost,port=38741 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:50,282 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:50,283 DEBUG [Thread-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-16T14:39:50,283 DEBUG [Thread-14 {}] ipc.FailedServers(52): Added failed server with address localhost:38741 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,283 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:50,284 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 38741 2024-11-16T14:39:50,284 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:50,284 INFO [Listener,port=38741 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=38741: stopping 2024-11-16T14:39:50,284 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:50,290 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=23 (was 19) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.util.concurrent.atomic.AtomicBoolean.get(AtomicBoolean.java:88) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:140) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) Potentially hanging thread: RpcServer idle connection scanner for port 38741 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Reader=2,bindAddress=localhost,port=38741 java.base@17.0.11/sun.nio.ch.IOUtil.drain(Native Method) java.base@17.0.11/sun.nio.ch.EventFD.reset(EventFD.java:52) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.clearInterrupt(EPollSelectorImpl.java:263) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.processEvents(EPollSelectorImpl.java:201) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:135) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Listener,port=38741 java.base@17.0.11/java.lang.ThreadLocal.get(ThreadLocal.java:163) app//org.apache.logging.log4j.spi.AbstractLogger.getRecursionDepthHolder(AbstractLogger.java:2166) app//org.apache.logging.log4j.spi.AbstractLogger.getRecursionDepth(AbstractLogger.java:2192) app//org.apache.logging.log4j.core.layout.AbstractStringLayout.getStringBuilder(AbstractStringLayout.java:129) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:225) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:252) - Thread LEAK? -, OpenFileDescriptor=292 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 330), ProcessCount=11 (was 11), AvailableMemoryMB=7239 (was 7282) 2024-11-16T14:39:50,297 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=7238 2024-11-16T14:39:50,308 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:50,308 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:50,314 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767990313,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:50,319 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:50,321 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=32981 2024-11-16T14:39:50,321 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:50,321 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:50,321 INFO [Listener,port=32981 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=32981: starting 2024-11-16T14:39:50,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@738a8f27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:50,323 DEBUG [Thread-17 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:50,323 DEBUG [Thread-17 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:32981 2024-11-16T14:39:50,324 DEBUG [Thread-17 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,331 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767990330,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,334 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:50,334 DEBUG [Reader=1,bindAddress=localhost,port=32981 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,336 DEBUG [Reader=1,bindAddress=localhost,port=32981 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,336 DEBUG [Reader=1,bindAddress=localhost,port=32981 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,338 DEBUG [Reader=1,bindAddress=localhost,port=32981 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:50,339 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:50,339 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:50,339 DEBUG [Reader=1,bindAddress=localhost,port=32981 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,340 DEBUG [Reader=1,bindAddress=localhost,port=32981 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,340 DEBUG [Reader=1,bindAddress=localhost,port=32981 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:50,340 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:50,341 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:50,341 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-16T14:39:50,341 DEBUG [Thread-17 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:50,341 DEBUG [Reader=1,bindAddress=localhost,port=32981 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,341 DEBUG [Reader=1,bindAddress=localhost,port=32981 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,342 DEBUG [Reader=1,bindAddress=localhost,port=32981 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,342 DEBUG [Reader=1,bindAddress=localhost,port=32981 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:50,342 INFO [Reader=1,bindAddress=localhost,port=32981 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:50,343 INFO [Reader=1,bindAddress=localhost,port=32981 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42498, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:50,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:50,372 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 32981 2024-11-16T14:39:50,372 INFO [Listener,port=32981 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=32981: stopping 2024-11-16T14:39:50,372 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:50,372 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:50,373 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:50,374 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39739 2024-11-16T14:39:50,375 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:50,375 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:50,375 INFO [Listener,port=39739 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39739: starting 2024-11-16T14:39:50,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@436542f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:50,376 DEBUG [Thread-20 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:50,376 DEBUG [Thread-20 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39739 2024-11-16T14:39:50,377 DEBUG [Thread-20 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,379 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:50,379 DEBUG [Reader=1,bindAddress=localhost,port=39739 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,380 DEBUG [Reader=1,bindAddress=localhost,port=39739 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,380 DEBUG [Reader=1,bindAddress=localhost,port=39739 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,382 DEBUG [Reader=1,bindAddress=localhost,port=39739 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:50,382 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:50,383 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:50,383 DEBUG [Reader=1,bindAddress=localhost,port=39739 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,383 DEBUG [Reader=1,bindAddress=localhost,port=39739 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,384 DEBUG [Reader=1,bindAddress=localhost,port=39739 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:50,384 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:50,384 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:50,384 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-16T14:39:50,385 DEBUG [Thread-20 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:50,384 DEBUG [Reader=1,bindAddress=localhost,port=39739 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,385 DEBUG [Reader=1,bindAddress=localhost,port=39739 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,385 DEBUG [Reader=1,bindAddress=localhost,port=39739 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,385 DEBUG [Reader=1,bindAddress=localhost,port=39739 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:50,386 INFO [Reader=1,bindAddress=localhost,port=39739 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:50,386 INFO [Reader=1,bindAddress=localhost,port=39739 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:32890, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:50,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:50,409 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39739 2024-11-16T14:39:50,409 INFO [Listener,port=39739 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39739: stopping 2024-11-16T14:39:50,410 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:50,410 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:50,410 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:50,412 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=43011 2024-11-16T14:39:50,415 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:50,415 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:50,415 INFO [Listener,port=43011 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=43011: starting 2024-11-16T14:39:50,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d3a9b95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:50,417 DEBUG [Thread-23 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:50,417 DEBUG [Thread-23 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43011 2024-11-16T14:39:50,419 DEBUG [Thread-23 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,421 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:50,421 DEBUG [Reader=1,bindAddress=localhost,port=43011 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,422 DEBUG [Reader=1,bindAddress=localhost,port=43011 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,423 DEBUG [Reader=1,bindAddress=localhost,port=43011 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,425 DEBUG [Reader=1,bindAddress=localhost,port=43011 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:50,425 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:50,426 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:50,426 DEBUG [Reader=1,bindAddress=localhost,port=43011 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,426 DEBUG [Reader=1,bindAddress=localhost,port=43011 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,427 DEBUG [Reader=1,bindAddress=localhost,port=43011 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:50,427 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:50,428 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:50,428 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-16T14:39:50,428 DEBUG [Reader=1,bindAddress=localhost,port=43011 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,428 DEBUG [Thread-23 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:50,429 DEBUG [Reader=1,bindAddress=localhost,port=43011 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,429 DEBUG [Reader=1,bindAddress=localhost,port=43011 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,429 DEBUG [Reader=1,bindAddress=localhost,port=43011 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:50,429 INFO [Reader=1,bindAddress=localhost,port=43011 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:50,430 INFO [Reader=1,bindAddress=localhost,port=43011 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34280, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:50,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:50,448 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 43011 2024-11-16T14:39:50,448 INFO [Listener,port=43011 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=43011: stopping 2024-11-16T14:39:50,448 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:50,448 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:50,449 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:50,450 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=36223 2024-11-16T14:39:50,451 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:50,451 INFO [Listener,port=36223 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=36223: starting 2024-11-16T14:39:50,451 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:50,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55ab6268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:50,452 DEBUG [Thread-26 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:50,453 DEBUG [Thread-26 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:36223 2024-11-16T14:39:50,454 DEBUG [Thread-26 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,455 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:50,456 DEBUG [Reader=1,bindAddress=localhost,port=36223 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,457 DEBUG [Reader=1,bindAddress=localhost,port=36223 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,457 DEBUG [Reader=1,bindAddress=localhost,port=36223 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,458 DEBUG [Reader=1,bindAddress=localhost,port=36223 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:50,458 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:50,459 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:50,459 DEBUG [Reader=1,bindAddress=localhost,port=36223 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,459 DEBUG [Reader=1,bindAddress=localhost,port=36223 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,460 DEBUG [Reader=1,bindAddress=localhost,port=36223 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:50,460 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:50,460 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:50,460 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-int 2024-11-16T14:39:50,460 DEBUG [Reader=1,bindAddress=localhost,port=36223 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,460 DEBUG [Thread-26 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:50,460 DEBUG [Reader=1,bindAddress=localhost,port=36223 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,461 DEBUG [Reader=1,bindAddress=localhost,port=36223 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,461 DEBUG [Reader=1,bindAddress=localhost,port=36223 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int 2024-11-16T14:39:50,461 INFO [Reader=1,bindAddress=localhost,port=36223 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:50,462 INFO [Reader=1,bindAddress=localhost,port=36223 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42186, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:50,474 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:50,475 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 36223 2024-11-16T14:39:50,475 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:50,475 INFO [Listener,port=36223 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=36223: stopping 2024-11-16T14:39:50,475 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:50,475 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:50,477 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39693 2024-11-16T14:39:50,477 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:50,477 INFO [Listener,port=39693 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39693: starting 2024-11-16T14:39:50,477 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:50,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c34501, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:50,479 DEBUG [Thread-29 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:50,479 DEBUG [Thread-29 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39693 2024-11-16T14:39:50,480 DEBUG [Thread-29 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,481 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:50,482 DEBUG [Reader=1,bindAddress=localhost,port=39693 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,482 DEBUG [Reader=1,bindAddress=localhost,port=39693 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,482 DEBUG [Reader=1,bindAddress=localhost,port=39693 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,484 DEBUG [Reader=1,bindAddress=localhost,port=39693 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:50,484 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:50,484 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:50,485 DEBUG [Reader=1,bindAddress=localhost,port=39693 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,485 DEBUG [Reader=1,bindAddress=localhost,port=39693 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,485 DEBUG [Reader=1,bindAddress=localhost,port=39693 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:50,485 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:50,486 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:50,486 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-16T14:39:50,486 DEBUG [Reader=1,bindAddress=localhost,port=39693 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,486 DEBUG [Reader=1,bindAddress=localhost,port=39693 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,486 DEBUG [Thread-29 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:50,487 DEBUG [Reader=1,bindAddress=localhost,port=39693 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,487 DEBUG [Reader=1,bindAddress=localhost,port=39693 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-16T14:39:50,487 INFO [Reader=1,bindAddress=localhost,port=39693 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:50,487 INFO [Reader=1,bindAddress=localhost,port=39693 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39494, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:50,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:50,504 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39693 2024-11-16T14:39:50,504 INFO [Listener,port=39693 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39693: stopping 2024-11-16T14:39:50,505 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:50,505 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:50,510 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.tryAdvance(StackStreamFactory.java:590) java.base@17.0.11/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129) java.base@17.0.11/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527) java.base@17.0.11/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513) java.base@17.0.11/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499) java.base@17.0.11/java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150) java.base@17.0.11/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) java.base@17.0.11/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647) app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$15(StackLocator.java:100) app//org.apache.logging.log4j.util.StackLocator$$Lambda$125/0x00007fe1f418c260.apply(Unknown Source) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243) java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499) app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97) app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121) app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 330), ProcessCount=11 (was 11), AvailableMemoryMB=7220 (was 7238) 2024-11-16T14:39:50,516 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=7220 2024-11-16T14:39:50,528 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:50,528 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:50,533 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767990533,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:50,538 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:50,540 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=42309 2024-11-16T14:39:50,541 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:50,541 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:50,541 INFO [Listener,port=42309 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=42309: starting 2024-11-16T14:39:50,541 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b601aef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:50,542 DEBUG [Thread-32 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T14:39:50,543 DEBUG [Thread-32 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42309 2024-11-16T14:39:50,547 DEBUG [BRPC Connection (1845770469) to localhost:42309 from testuser {}] ipc.RpcConnection(424): process preamble call response with response type null 2024-11-16T14:39:50,547 WARN [BRPC Connection (1845770469) to localhost:42309 from testuser {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call 2024-11-16T14:39:50,548 DEBUG [BRPC Connection (1845770469) to localhost:42309 from testuser {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes 2024-11-16T14:39:50,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:50,549 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 42309 2024-11-16T14:39:50,549 INFO [Listener,port=42309 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=42309: stopping 2024-11-16T14:39:50,549 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:50,550 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:50,557 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 19), OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 330), ProcessCount=11 (was 11), AvailableMemoryMB=7210 (was 7220) 2024-11-16T14:39:50,562 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=7208 2024-11-16T14:39:50,573 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:50,573 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:50,579 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767990578,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:50,583 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:50,591 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34319 2024-11-16T14:39:50,591 WARN [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! ********* 2024-11-16T14:39:50,591 WARN [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients 2024-11-16T14:39:50,591 WARN [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true). 2024-11-16T14:39:50,591 WARN [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user 2024-11-16T14:39:50,592 WARN [Time-limited test {}] ipc.RpcServer(380): impersonation is possible! 2024-11-16T14:39:50,592 WARN [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible, 2024-11-16T14:39:50,592 WARN [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml 2024-11-16T14:39:50,592 WARN [Time-limited test {}] ipc.RpcServer(383): **************************** 2024-11-16T14:39:50,592 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:50,592 INFO [Listener,port=34319 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34319: starting 2024-11-16T14:39:50,592 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:50,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f2f4a66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:50,596 DEBUG [Thread-35 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T14:39:50,597 DEBUG [Thread-35 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34319 2024-11-16T14:39:50,598 WARN [Reader=1,bindAddress=localhost,port=34319 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1 2024-11-16T14:39:50,598 INFO [Reader=1,bindAddress=localhost,port=34319 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T14:39:50,610 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:50,610 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34319 2024-11-16T14:39:50,610 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:50,610 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:50,611 INFO [Listener,port=34319 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34319: stopping 2024-11-16T14:39:50,615 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 19), OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 330), ProcessCount=11 (was 11), AvailableMemoryMB=7197 (was 7208) 2024-11-16T14:39:50,620 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=7196 2024-11-16T14:39:50,631 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:50,631 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:50,636 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767990636,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:50,640 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:50,642 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34717 2024-11-16T14:39:50,645 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:50,645 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:50,645 INFO [Listener,port=34717 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34717: starting 2024-11-16T14:39:50,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@563b3d49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:50,647 DEBUG [Thread-38 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:50,647 DEBUG [Thread-38 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34717 2024-11-16T14:39:50,648 DEBUG [Thread-38 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,654 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767990654,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,656 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:50,656 DEBUG [Reader=1,bindAddress=localhost,port=34717 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,658 DEBUG [Reader=1,bindAddress=localhost,port=34717 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,658 DEBUG [Reader=1,bindAddress=localhost,port=34717 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,660 DEBUG [Reader=1,bindAddress=localhost,port=34717 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:50,660 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:50,660 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:50,661 DEBUG [Reader=1,bindAddress=localhost,port=34717 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,661 DEBUG [Reader=1,bindAddress=localhost,port=34717 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,661 DEBUG [Reader=1,bindAddress=localhost,port=34717 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:50,661 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:50,662 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:50,662 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-16T14:39:50,662 DEBUG [Reader=1,bindAddress=localhost,port=34717 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,662 DEBUG [Thread-38 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:50,662 DEBUG [Reader=1,bindAddress=localhost,port=34717 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,662 DEBUG [Reader=1,bindAddress=localhost,port=34717 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,662 DEBUG [Reader=1,bindAddress=localhost,port=34717 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:50,663 INFO [Reader=1,bindAddress=localhost,port=34717 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:50,663 INFO [Reader=1,bindAddress=localhost,port=34717 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50212, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:50,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:50,674 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34717 2024-11-16T14:39:50,674 INFO [Listener,port=34717 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34717: stopping 2024-11-16T14:39:50,676 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:50,676 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:50,681 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.lang.StackStreamFactory$FrameBuffer.(StackStreamFactory.java:794) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser$StackFrameBuffer.(StackStreamFactory.java:451) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.initFrameBuffer(StackStreamFactory.java:541) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:368) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243) java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499) app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97) app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121) app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) - Thread LEAK? -, OpenFileDescriptor=292 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 330), ProcessCount=11 (was 11), AvailableMemoryMB=7179 (was 7196) 2024-11-16T14:39:50,688 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=7178 2024-11-16T14:39:50,704 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:50,705 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:50,710 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767990710,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:50,713 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:50,714 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45005 2024-11-16T14:39:50,715 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:50,715 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:50,715 INFO [Listener,port=45005 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45005: starting 2024-11-16T14:39:50,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d919fa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:50,718 DEBUG [Thread-41 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:50,719 DEBUG [Thread-41 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45005 2024-11-16T14:39:50,720 DEBUG [Thread-41 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,727 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767990727,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,729 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:50,729 DEBUG [Reader=1,bindAddress=localhost,port=45005 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,730 DEBUG [Reader=1,bindAddress=localhost,port=45005 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,730 DEBUG [Reader=1,bindAddress=localhost,port=45005 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,732 DEBUG [Reader=1,bindAddress=localhost,port=45005 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:50,732 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:50,732 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:50,733 DEBUG [Reader=1,bindAddress=localhost,port=45005 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,733 DEBUG [Reader=1,bindAddress=localhost,port=45005 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,733 DEBUG [Reader=1,bindAddress=localhost,port=45005 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:50,733 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:50,734 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:50,734 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-16T14:39:50,734 DEBUG [Reader=1,bindAddress=localhost,port=45005 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,734 DEBUG [Thread-41 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:50,734 DEBUG [Reader=1,bindAddress=localhost,port=45005 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,734 DEBUG [Reader=1,bindAddress=localhost,port=45005 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,734 DEBUG [Reader=1,bindAddress=localhost,port=45005 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-16T14:39:50,735 INFO [Reader=1,bindAddress=localhost,port=45005 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:50,735 INFO [Reader=1,bindAddress=localhost,port=45005 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40174, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:50,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:50,750 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45005 2024-11-16T14:39:50,750 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:50,750 INFO [Listener,port=45005 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45005: stopping 2024-11-16T14:39:50,750 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:50,751 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:50,759 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=33433 2024-11-16T14:39:50,760 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:50,762 INFO [Listener,port=33433 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=33433: starting 2024-11-16T14:39:50,761 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:50,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e25d098, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:50,766 DEBUG [Thread-44 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:50,767 DEBUG [Thread-44 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:33433 2024-11-16T14:39:50,768 DEBUG [Thread-44 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,770 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:50,770 DEBUG [Reader=1,bindAddress=localhost,port=33433 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,771 DEBUG [Reader=1,bindAddress=localhost,port=33433 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,771 DEBUG [Reader=1,bindAddress=localhost,port=33433 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,773 DEBUG [Reader=1,bindAddress=localhost,port=33433 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:50,773 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:50,773 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:50,774 DEBUG [Reader=1,bindAddress=localhost,port=33433 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,774 DEBUG [Reader=1,bindAddress=localhost,port=33433 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,774 DEBUG [Reader=1,bindAddress=localhost,port=33433 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:50,774 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:50,775 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:50,775 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-16T14:39:50,775 DEBUG [Reader=1,bindAddress=localhost,port=33433 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:50,775 DEBUG [Thread-44 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:50,775 DEBUG [Reader=1,bindAddress=localhost,port=33433 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:50,775 DEBUG [Reader=1,bindAddress=localhost,port=33433 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:50,776 DEBUG [Reader=1,bindAddress=localhost,port=33433 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-16T14:39:50,776 INFO [Reader=1,bindAddress=localhost,port=33433 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:50,776 INFO [Reader=1,bindAddress=localhost,port=33433 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39998, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:52,778 ERROR [Thread-44 {}] ipc.BlockingRpcConnection(646): Can't get the connection header response for rpc timeout, please check if server has the correct configuration to support the additional function. java.net.SocketTimeoutException: 2000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39998 remote=localhost/127.0.0.1:33433] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:163) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:132) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:201) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.security.SaslInputStream.readMoreData(SaslInputStream.java:97) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.security.SaslInputStream.read(SaslInputStream.java:260) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:263) ~[?:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.processResponseForConnectionHeader(BlockingRpcConnection.java:630) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:570) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-16T14:39:52,779 DEBUG [Thread-44 {}] ipc.FailedServers(52): Added failed server with address localhost:33433 to list caused by java.io.IOException: Timeout while waiting connection header response 2024-11-16T14:39:52,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:52,779 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 33433 2024-11-16T14:39:52,780 INFO [Listener,port=33433 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=33433: stopping 2024-11-16T14:39:52,780 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:52,780 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:52,786 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.core.layout.TextEncoderHelper.writeEncodedText(TextEncoderHelper.java:82) app//org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeText(TextEncoderHelper.java:65) app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:68) app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:32) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:227) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 330), ProcessCount=11 (was 11), AvailableMemoryMB=6982 (was 7178) 2024-11-16T14:39:52,792 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=6982 2024-11-16T14:39:52,808 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:52,808 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:52,813 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767992812,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:52,817 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:52,819 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39487 2024-11-16T14:39:52,823 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:52,826 INFO [Listener,port=39487 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39487: starting 2024-11-16T14:39:52,826 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:52,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e064221, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:52,828 DEBUG [Thread-47 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:52,828 DEBUG [Thread-47 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39487 2024-11-16T14:39:52,830 DEBUG [Thread-47 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:52,836 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767992835,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:52,838 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:52,838 DEBUG [Reader=1,bindAddress=localhost,port=39487 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:52,839 DEBUG [Reader=1,bindAddress=localhost,port=39487 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:52,839 DEBUG [Reader=1,bindAddress=localhost,port=39487 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:52,841 DEBUG [Reader=1,bindAddress=localhost,port=39487 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:52,841 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:52,842 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:52,842 DEBUG [Reader=1,bindAddress=localhost,port=39487 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:52,842 DEBUG [Reader=1,bindAddress=localhost,port=39487 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:52,843 DEBUG [Reader=1,bindAddress=localhost,port=39487 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:52,843 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:52,843 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:52,843 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-16T14:39:52,843 DEBUG [Reader=1,bindAddress=localhost,port=39487 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:52,843 DEBUG [Thread-47 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:52,844 DEBUG [Reader=1,bindAddress=localhost,port=39487 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:52,844 DEBUG [Reader=1,bindAddress=localhost,port=39487 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:52,844 DEBUG [Reader=1,bindAddress=localhost,port=39487 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:52,844 INFO [Reader=1,bindAddress=localhost,port=39487 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:52,844 INFO [Reader=1,bindAddress=localhost,port=39487 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50456, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:52,853 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:52,853 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39487 2024-11-16T14:39:52,853 INFO [Listener,port=39487 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39487: stopping 2024-11-16T14:39:52,853 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:52,854 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:52,858 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150) java.base@17.0.11/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) java.base@17.0.11/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647) app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$15(StackLocator.java:100) app//org.apache.logging.log4j.util.StackLocator$$Lambda$125/0x00007fe1f418c260.apply(Unknown Source) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243) java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499) app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97) app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121) app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 330), ProcessCount=11 (was 11), AvailableMemoryMB=6962 (was 6982) 2024-11-16T14:39:52,863 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=6962 2024-11-16T14:39:52,874 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:52,874 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:52,878 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767992877,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:52,880 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:52,882 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44165 2024-11-16T14:39:52,882 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:52,882 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:52,882 INFO [Listener,port=44165 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44165: starting 2024-11-16T14:39:52,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@81a5138, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:52,883 DEBUG [Thread-50 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:52,883 DEBUG [Thread-50 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44165 2024-11-16T14:39:52,884 DEBUG [Thread-50 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:52,889 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767992889,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:52,891 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:52,891 DEBUG [Reader=1,bindAddress=localhost,port=44165 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:52,892 DEBUG [Reader=1,bindAddress=localhost,port=44165 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:52,892 DEBUG [Reader=1,bindAddress=localhost,port=44165 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:52,893 DEBUG [Reader=1,bindAddress=localhost,port=44165 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:52,893 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:52,893 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:52,893 DEBUG [Reader=1,bindAddress=localhost,port=44165 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:52,893 DEBUG [Reader=1,bindAddress=localhost,port=44165 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:52,894 DEBUG [Reader=1,bindAddress=localhost,port=44165 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:52,894 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:52,894 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:52,894 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-16T14:39:52,894 DEBUG [Reader=1,bindAddress=localhost,port=44165 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:52,894 DEBUG [Thread-50 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:52,894 DEBUG [Reader=1,bindAddress=localhost,port=44165 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:52,894 DEBUG [Reader=1,bindAddress=localhost,port=44165 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:52,894 DEBUG [Reader=1,bindAddress=localhost,port=44165 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:52,895 INFO [Reader=1,bindAddress=localhost,port=44165 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:52,895 INFO [Reader=1,bindAddress=localhost,port=44165 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43242, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:52,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:52,903 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44165 2024-11-16T14:39:52,903 INFO [Listener,port=44165 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44165: stopping 2024-11-16T14:39:52,903 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:52,904 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:52,908 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2021) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 330), ProcessCount=11 (was 11), AvailableMemoryMB=6946 (was 6962) 2024-11-16T14:39:52,915 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=6945 2024-11-16T14:39:52,926 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:52,926 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:52,930 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767992930,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:52,933 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:52,935 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=35871 2024-11-16T14:39:52,935 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:52,935 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:52,935 INFO [Listener,port=35871 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=35871: starting 2024-11-16T14:39:52,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@94ded4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:52,937 DEBUG [Thread-53 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:52,937 DEBUG [Thread-53 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:35871 2024-11-16T14:39:52,938 DEBUG [Thread-53 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:52,944 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767992943,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:52,946 DEBUG [Thread-53 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:52,946 DEBUG [Thread-53 {}] security.HBaseSaslRpcClient(113): Server asks us to fall back to simple auth. 2024-11-16T14:39:52,946 DEBUG [Thread-53 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:52,946 INFO [Reader=1,bindAddress=localhost,port=35871 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T14:39:52,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:52,955 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 35871 2024-11-16T14:39:52,955 INFO [Listener,port=35871 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=35871: stopping 2024-11-16T14:39:52,955 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:52,956 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:52,960 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=21 (was 19) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.core.pattern.DatePatternConverter.format(DatePatternConverter.java:265) app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:384) app//org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:240) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:225) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) Potentially hanging thread: Reader=2,bindAddress=localhost,port=35871 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 330), ProcessCount=11 (was 11), AvailableMemoryMB=6927 (was 6945) 2024-11-16T14:39:52,966 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=6926 2024-11-16T14:39:52,977 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:52,978 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:52,982 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767992981,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:52,984 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:52,996 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:52,999 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T14:39:53,029 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1565 (auto-detected) 2024-11-16T14:39:53,030 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-16T14:39:53,052 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35561 2024-11-16T14:39:53,052 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:53,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@892ead0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:53,053 DEBUG [Thread-54 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:53,054 DEBUG [Thread-54 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:35561 2024-11-16T14:39:53,055 DEBUG [Thread-54 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,061 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767993061,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,063 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:53,088 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,088 DEBUG [NettyRpcServer-2-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,090 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:53,093 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:53,094 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:53,095 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,095 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:53,095 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:53,096 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:53,096 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-16T14:39:53,096 DEBUG [Thread-54 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:53,096 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,096 DEBUG [NettyRpcServer-2-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,096 DEBUG [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-16T14:39:53,097 INFO [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:53,102 INFO [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37730, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:53,102 DEBUG [Thread-54 {}] ipc.BlockingRpcConnection(634): Length of response for connection header:93 2024-11-16T14:39:53,104 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:254 2024-11-16T14:39:53,105 DEBUG [BRPC Connection (1845770469) to localhost:35561 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-16T14:39:53,108 DEBUG [BRPC Connection (1845770469) to localhost:35561 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:240 2024-11-16T14:39:53,109 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:2267 2024-11-16T14:39:53,109 DEBUG [BRPC Connection (1845770469) to localhost:35561 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-16T14:39:53,111 DEBUG [BRPC Connection (1845770469) to localhost:35561 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:2253 2024-11-16T14:39:53,113 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:4117 2024-11-16T14:39:53,114 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:18161 2024-11-16T14:39:53,116 DEBUG [BRPC Connection (1845770469) to localhost:35561 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-16T14:39:53,123 DEBUG [BRPC Connection (1845770469) to localhost:35561 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:22264 2024-11-16T14:39:53,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:53,124 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35561 2024-11-16T14:39:53,134 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=21 (was 19) Potentially hanging thread: NettyRpcServer-2-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-2-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=299 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 330), ProcessCount=11 (was 11), AvailableMemoryMB=6889 (was 6926) 2024-11-16T14:39:53,146 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=21, OpenFileDescriptor=299, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=6887 2024-11-16T14:39:53,159 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:53,159 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:53,163 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767993163,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:53,166 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:53,167 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:53,169 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36577 2024-11-16T14:39:53,170 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:53,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23f3313c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-16T14:39:53,171 DEBUG [Thread-55 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:53,172 DEBUG [Thread-55 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:36577 2024-11-16T14:39:53,173 DEBUG [Thread-55 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,179 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767993179,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,181 DEBUG [Thread-55 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:53,188 DEBUG [Thread-55 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-16T14:39:53,189 DEBUG [Thread-55 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:36577 org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections. at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:111) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-16T14:39:53,190 WARN [Thread-55 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731767993164 2024-11-16T14:39:53,288 DEBUG [Thread-55 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,290 DEBUG [Thread-55 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:53,291 DEBUG [Thread-55 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-16T14:39:53,291 DEBUG [Thread-55 {}] ipc.FailedServers(52): Added failed server with address localhost:36577 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:53,292 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36577 2024-11-16T14:39:53,298 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=24 (was 21) Potentially hanging thread: NettyRpcServer-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=308 (was 299) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 330), ProcessCount=11 (was 11), AvailableMemoryMB=6865 (was 6887) 2024-11-16T14:39:53,305 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=24, OpenFileDescriptor=308, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=6865 2024-11-16T14:39:53,324 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:53,324 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:53,331 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767993331,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:53,334 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:53,335 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:53,336 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37151 2024-11-16T14:39:53,337 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:53,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@412d9ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:53,339 DEBUG [Thread-56 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:53,339 DEBUG [Thread-56 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37151 2024-11-16T14:39:53,341 DEBUG [Thread-56 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,346 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767993346,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,350 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:53,351 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,351 DEBUG [NettyRpcServer-4-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,353 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:53,354 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:53,354 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:53,354 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,354 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:53,355 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:53,355 DEBUG [Thread-56 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-16T14:39:53,355 DEBUG [Thread-56 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:37151 javax.security.sasl.SaslException: No common protection layer between client and server at com.sun.security.sasl.gsskerb.GssKrb5Client.doFinalHandshake(GssKrb5Client.java:270) ~[jdk.security.jgss:?] at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:203) ~[jdk.security.jgss:?] at org.apache.hadoop.hbase.security.AbstractHBaseSaslRpcClient.evaluateChallenge(AbstractHBaseSaslRpcClient.java:105) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:126) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-16T14:39:53,356 WARN [Thread-56 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731767993333 2024-11-16T14:39:53,453 DEBUG [Thread-56 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,454 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:53,454 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,454 DEBUG [NettyRpcServer-4-3 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,457 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:53,457 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:53,458 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:53,458 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,458 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:53,459 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:53,459 DEBUG [Thread-56 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-16T14:39:53,459 DEBUG [Thread-56 {}] ipc.FailedServers(52): Added failed server with address localhost:37151 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:53,460 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37151 2024-11-16T14:39:53,466 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=27 (was 24) Potentially hanging thread: NettyRpcServer-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=317 (was 308) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 330), ProcessCount=11 (was 11), AvailableMemoryMB=6843 (was 6865) 2024-11-16T14:39:53,472 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=27, OpenFileDescriptor=317, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=6842 2024-11-16T14:39:53,483 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:53,484 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:53,488 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767993488,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:53,490 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:53,491 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:53,492 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37763 2024-11-16T14:39:53,492 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:53,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27a8d523, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:53,493 DEBUG [Thread-57 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:53,493 DEBUG [Thread-57 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37763 2024-11-16T14:39:53,495 DEBUG [Thread-57 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,501 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767993501,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,503 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:53,503 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,503 DEBUG [NettyRpcServer-5-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,505 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:53,505 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:53,506 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:53,506 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,506 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:53,506 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:53,507 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:53,507 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-16T14:39:53,507 DEBUG [Thread-57 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:53,507 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,507 DEBUG [NettyRpcServer-5-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,507 DEBUG [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:53,507 INFO [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:53,508 INFO [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46040, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:53,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:53,518 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37763 2024-11-16T14:39:53,520 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:53,520 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:53,521 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40717 2024-11-16T14:39:53,521 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:53,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@739dcf3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:53,523 DEBUG [Thread-58 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:53,523 DEBUG [Thread-58 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40717 2024-11-16T14:39:53,524 DEBUG [Thread-58 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,526 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:53,526 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,526 DEBUG [NettyRpcServer-6-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,527 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:53,528 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:53,528 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:53,528 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,529 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:53,529 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:53,529 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:53,529 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-16T14:39:53,529 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,529 DEBUG [Thread-58 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:53,530 DEBUG [NettyRpcServer-6-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,530 DEBUG [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:53,530 INFO [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:53,530 INFO [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36826, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:53,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:53,539 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40717 2024-11-16T14:39:53,541 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:53,541 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:53,542 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42023 2024-11-16T14:39:53,542 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:53,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6816a9ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:53,543 DEBUG [Thread-59 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:53,544 DEBUG [Thread-59 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42023 2024-11-16T14:39:53,545 DEBUG [Thread-59 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,546 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:53,548 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,548 DEBUG [NettyRpcServer-7-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,549 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:53,550 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:53,550 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:53,551 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,551 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:53,551 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:53,551 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:53,551 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-16T14:39:53,552 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,552 DEBUG [Thread-59 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:53,552 DEBUG [NettyRpcServer-7-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,552 DEBUG [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:53,552 INFO [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:53,553 INFO [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56964, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:53,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:53,568 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42023 2024-11-16T14:39:53,569 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:53,569 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:53,570 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35997 2024-11-16T14:39:53,571 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:53,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a5d9da7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:53,572 DEBUG [Thread-60 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:53,572 DEBUG [Thread-60 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:35997 2024-11-16T14:39:53,573 DEBUG [Thread-60 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,575 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:53,575 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,575 DEBUG [NettyRpcServer-8-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,577 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:53,578 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:53,578 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:53,578 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,578 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:53,579 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:53,579 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:53,579 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-int 2024-11-16T14:39:53,579 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,579 DEBUG [Thread-60 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:53,579 DEBUG [NettyRpcServer-8-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,580 DEBUG [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int 2024-11-16T14:39:53,580 INFO [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:53,580 INFO [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49286, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:53,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:53,591 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35997 2024-11-16T14:39:53,592 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:53,592 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:53,593 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41321 2024-11-16T14:39:53,593 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:53,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab5aa0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:53,595 DEBUG [Thread-61 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:53,595 DEBUG [Thread-61 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:41321 2024-11-16T14:39:53,596 DEBUG [Thread-61 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,597 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:53,597 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,597 DEBUG [NettyRpcServer-9-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,599 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:53,599 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:53,600 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:53,600 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,600 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:53,600 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:53,601 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:53,601 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-16T14:39:53,601 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,601 DEBUG [Thread-61 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:53,601 DEBUG [NettyRpcServer-9-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,601 DEBUG [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-16T14:39:53,601 INFO [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:53,602 INFO [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46048, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:53,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:53,612 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41321 2024-11-16T14:39:53,619 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=37 (was 27) Potentially hanging thread: NettyRpcServer-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=362 (was 317) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 330), ProcessCount=11 (was 11), AvailableMemoryMB=6813 (was 6842) 2024-11-16T14:39:53,625 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=37, OpenFileDescriptor=362, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=6811 2024-11-16T14:39:53,641 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:53,641 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:53,645 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767993645,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:53,647 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:53,647 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:53,649 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40173 2024-11-16T14:39:53,649 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:53,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dcaf058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:53,650 DEBUG [Thread-62 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T14:39:53,650 DEBUG [Thread-62 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40173 2024-11-16T14:39:53,654 DEBUG [BRPC Connection (1845770469) to localhost:40173 from testuser {}] ipc.RpcConnection(424): process preamble call response with response type null 2024-11-16T14:39:53,654 WARN [BRPC Connection (1845770469) to localhost:40173 from testuser {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call 2024-11-16T14:39:53,654 DEBUG [BRPC Connection (1845770469) to localhost:40173 from testuser {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes 2024-11-16T14:39:53,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:53,655 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40173 2024-11-16T14:39:53,662 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=39 (was 37) Potentially hanging thread: NettyRpcServer-10-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-10-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=371 (was 362) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 330), ProcessCount=11 (was 11), AvailableMemoryMB=6799 (was 6811) 2024-11-16T14:39:53,668 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=39, OpenFileDescriptor=371, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=6798 2024-11-16T14:39:53,682 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:53,683 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:53,688 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767993687,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:53,691 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:53,692 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:53,693 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40725 2024-11-16T14:39:53,693 WARN [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! ********* 2024-11-16T14:39:53,693 WARN [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients 2024-11-16T14:39:53,693 WARN [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true). 2024-11-16T14:39:53,693 WARN [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user 2024-11-16T14:39:53,693 WARN [Time-limited test {}] ipc.RpcServer(380): impersonation is possible! 2024-11-16T14:39:53,693 WARN [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible, 2024-11-16T14:39:53,693 WARN [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml 2024-11-16T14:39:53,693 WARN [Time-limited test {}] ipc.RpcServer(383): **************************** 2024-11-16T14:39:53,694 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:53,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a83f771, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:53,695 DEBUG [Thread-63 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T14:39:53,695 DEBUG [Thread-63 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40725 2024-11-16T14:39:53,701 WARN [NettyRpcServer-11-2 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1 2024-11-16T14:39:53,701 INFO [NettyRpcServer-11-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:52550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T14:39:53,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:53,709 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40725 2024-11-16T14:39:53,719 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=41 (was 39) Potentially hanging thread: NettyRpcServer-11-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-11-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=380 (was 371) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 330), ProcessCount=11 (was 11), AvailableMemoryMB=6784 (was 6798) 2024-11-16T14:39:53,724 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=41, OpenFileDescriptor=380, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=6783 2024-11-16T14:39:53,735 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:53,735 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:53,739 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767993739,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:53,741 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:53,741 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:53,742 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45049 2024-11-16T14:39:53,742 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:53,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e172df4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:53,743 DEBUG [Thread-64 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:53,744 DEBUG [Thread-64 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45049 2024-11-16T14:39:53,745 DEBUG [Thread-64 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,750 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767993750,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,752 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:53,752 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,752 DEBUG [NettyRpcServer-12-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,754 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:53,755 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:53,755 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:53,756 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,756 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:53,756 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:53,756 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:53,757 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-16T14:39:53,757 DEBUG [Thread-64 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:53,757 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,757 DEBUG [NettyRpcServer-12-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,757 DEBUG [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:53,757 INFO [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:53,758 INFO [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:45370, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:53,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:53,766 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45049 2024-11-16T14:39:53,772 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=43 (was 41) Potentially hanging thread: NettyRpcServer-12-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-12-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=389 (was 380) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 330), ProcessCount=11 (was 11), AvailableMemoryMB=6765 (was 6783) 2024-11-16T14:39:53,778 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=43, OpenFileDescriptor=389, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=6764 2024-11-16T14:39:53,790 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:53,791 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:53,795 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767993794,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:53,797 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:53,798 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:53,799 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34653 2024-11-16T14:39:53,799 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:53,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71c59e2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:53,800 DEBUG [Thread-65 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:53,801 DEBUG [Thread-65 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34653 2024-11-16T14:39:53,802 DEBUG [Thread-65 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,806 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767993806,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,808 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:53,809 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,809 DEBUG [NettyRpcServer-13-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,811 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:53,811 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:53,812 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:53,812 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,812 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:53,812 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:53,813 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:53,813 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-16T14:39:53,813 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,813 DEBUG [Thread-65 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:53,813 DEBUG [NettyRpcServer-13-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,813 DEBUG [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-16T14:39:53,813 INFO [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:53,814 INFO [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39920, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:53,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:53,824 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34653 2024-11-16T14:39:53,825 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:53,825 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:53,826 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35739 2024-11-16T14:39:53,827 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:53,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a14eaea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:53,828 DEBUG [Thread-66 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:53,828 DEBUG [Thread-66 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:35739 2024-11-16T14:39:53,829 DEBUG [Thread-66 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,831 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:53,833 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,833 DEBUG [NettyRpcServer-14-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,835 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:53,836 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:53,836 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:53,836 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,837 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:53,837 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:53,837 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:53,837 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-16T14:39:53,838 DEBUG [Thread-66 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:53,840 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:53,840 DEBUG [NettyRpcServer-14-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:53,840 DEBUG [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-16T14:39:53,840 INFO [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:53,841 INFO [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48640, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:55,840 ERROR [Thread-66 {}] ipc.BlockingRpcConnection(646): Can't get the connection header response for rpc timeout, please check if server has the correct configuration to support the additional function. java.net.SocketTimeoutException: 2000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:48640 remote=localhost/127.0.0.1:35739] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:163) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:132) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:201) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.security.SaslInputStream.readMoreData(SaslInputStream.java:97) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.security.SaslInputStream.read(SaslInputStream.java:260) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:263) ~[?:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.processResponseForConnectionHeader(BlockingRpcConnection.java:630) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:570) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-16T14:39:55,841 DEBUG [Thread-66 {}] ipc.FailedServers(52): Added failed server with address localhost:35739 to list caused by java.io.IOException: Timeout while waiting connection header response 2024-11-16T14:39:55,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:55,843 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35739 2024-11-16T14:39:55,853 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=47 (was 43) Potentially hanging thread: NettyRpcServer-13-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-14-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-13-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-14-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=407 (was 389) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 330), ProcessCount=11 (was 11), AvailableMemoryMB=6729 (was 6764) 2024-11-16T14:39:55,859 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=47, OpenFileDescriptor=407, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6729 2024-11-16T14:39:55,870 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:55,870 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:55,874 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767995873,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:55,876 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:55,876 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:55,877 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46153 2024-11-16T14:39:55,877 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:55,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22ce1646, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:55,879 DEBUG [Thread-67 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:55,879 DEBUG [Thread-67 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:46153 2024-11-16T14:39:55,880 DEBUG [Thread-67 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:55,886 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767995885,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:55,887 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:55,887 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:55,887 DEBUG [NettyRpcServer-15-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:55,889 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:55,889 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:55,890 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:55,890 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:55,890 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:55,891 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:55,891 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:55,891 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-16T14:39:55,891 DEBUG [Thread-67 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:55,891 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:55,891 DEBUG [NettyRpcServer-15-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:55,892 DEBUG [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:55,892 INFO [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:55,892 INFO [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35974, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:55,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:55,900 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46153 2024-11-16T14:39:55,906 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=49 (was 47) Potentially hanging thread: NettyRpcServer-15-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-15-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=416 (was 407) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6712 (was 6729) 2024-11-16T14:39:55,912 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=49, OpenFileDescriptor=416, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6712 2024-11-16T14:39:55,927 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:55,927 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:55,930 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767995930,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:55,933 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:55,933 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:55,934 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45691 2024-11-16T14:39:55,934 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:55,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6ec393, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:55,935 DEBUG [Thread-68 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:55,935 DEBUG [Thread-68 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45691 2024-11-16T14:39:55,936 DEBUG [Thread-68 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:55,942 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767995941,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:55,943 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:55,943 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:55,944 DEBUG [NettyRpcServer-16-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:55,945 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:55,946 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-16T14:39:55,946 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-16T14:39:55,946 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:55,947 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:55,947 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-16T14:39:55,947 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-16T14:39:55,947 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-16T14:39:55,947 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:55,947 DEBUG [Thread-68 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:55,947 DEBUG [NettyRpcServer-16-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:55,948 DEBUG [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:55,948 INFO [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:55,948 INFO [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49572, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:55,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:55,957 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45691 2024-11-16T14:39:55,963 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=51 (was 49) Potentially hanging thread: NettyRpcServer-16-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-16-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=425 (was 416) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6695 (was 6712) 2024-11-16T14:39:55,969 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=51, OpenFileDescriptor=425, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6693 2024-11-16T14:39:55,982 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:55,982 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:55,987 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767995986,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:55,989 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:55,989 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:55,991 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41621 2024-11-16T14:39:55,991 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:55,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d8812a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:55,992 DEBUG [Thread-69 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:55,993 DEBUG [Thread-69 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:41621 2024-11-16T14:39:55,994 DEBUG [Thread-69 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:55,999 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767995998,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,000 DEBUG [Thread-69 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-16T14:39:56,000 DEBUG [Thread-69 {}] security.HBaseSaslRpcClient(113): Server asks us to fall back to simple auth. 2024-11-16T14:39:56,000 DEBUG [Thread-69 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:56,001 INFO [NettyRpcServer-17-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T14:39:56,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:56,009 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41621 2024-11-16T14:39:56,016 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=53 (was 51) Potentially hanging thread: NettyRpcServer-17-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-17-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=434 (was 425) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6675 (was 6693) 2024-11-16T14:39:56,022 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=53, OpenFileDescriptor=434, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6675 2024-11-16T14:39:56,033 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:56,033 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:56,036 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767996036,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:56,038 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:56,039 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45515 2024-11-16T14:39:56,040 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:56,042 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:56,042 INFO [Listener,port=45515 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45515: starting 2024-11-16T14:39:56,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f795f9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:56,046 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-16T14:39:56,046 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-16T14:39:56,050 DEBUG [Thread-72 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:56,063 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,068 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767996068,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,070 DEBUG [Reader=1,bindAddress=localhost,port=45515 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,071 DEBUG [Reader=1,bindAddress=localhost,port=45515 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,071 DEBUG [Reader=1,bindAddress=localhost,port=45515 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,072 DEBUG [Reader=1,bindAddress=localhost,port=45515 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:56,074 DEBUG [Reader=1,bindAddress=localhost,port=45515 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,074 DEBUG [Reader=1,bindAddress=localhost,port=45515 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,074 DEBUG [Reader=1,bindAddress=localhost,port=45515 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:56,074 DEBUG [Reader=1,bindAddress=localhost,port=45515 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,074 DEBUG [Reader=1,bindAddress=localhost,port=45515 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,075 DEBUG [Reader=1,bindAddress=localhost,port=45515 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,075 DEBUG [Reader=1,bindAddress=localhost,port=45515 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-16T14:39:56,075 INFO [Reader=1,bindAddress=localhost,port=45515 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:56,081 INFO [Reader=1,bindAddress=localhost,port=45515 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59092, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:56,082 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:56,095 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:56,096 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45515 2024-11-16T14:39:56,096 INFO [Listener,port=45515 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45515: stopping 2024-11-16T14:39:56,096 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:56,097 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:56,102 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=54 (was 53) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-1 app//org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel.eventLoop(AbstractNioChannel.java:48) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.executor(AbstractChannelHandlerContext.java:132) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.skipContext(AbstractChannelHandlerContext.java:1094) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.findContextInbound(AbstractChannelHandlerContext.java:1073) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelUnregistered(AbstractChannelHandlerContext.java:188) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelUnregistered(DefaultChannelPipeline.java:1335) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:215) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:195) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelUnregistered(DefaultChannelPipeline.java:770) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:819) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 434) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6653 (was 6675) 2024-11-16T14:39:56,108 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=54, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6653 2024-11-16T14:39:56,120 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:56,120 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:56,124 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767996124,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:56,126 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:56,127 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34041 2024-11-16T14:39:56,129 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:56,130 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:56,130 INFO [Listener,port=34041 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34041: starting 2024-11-16T14:39:56,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fcbe7d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-16T14:39:56,131 DEBUG [Thread-75 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:56,133 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,138 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767996138,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,140 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-16T14:39:56,141 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.FailedServers(52): Added failed server with address localhost:34041 to list caused by org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections. 2024-11-16T14:39:56,142 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:56,142 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34041 2024-11-16T14:39:56,142 INFO [Listener,port=34041 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34041: stopping 2024-11-16T14:39:56,143 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:56,143 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:56,149 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=57 (was 54) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243) java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499) app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97) app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121) app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) Potentially hanging thread: Reader=0,bindAddress=localhost,port=34041 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6636 (was 6653) 2024-11-16T14:39:56,156 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=55, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6636 2024-11-16T14:39:56,167 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:56,167 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:56,171 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767996171,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:56,173 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:56,174 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=36727 2024-11-16T14:39:56,175 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:56,175 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:56,175 INFO [Listener,port=36727 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=36727: starting 2024-11-16T14:39:56,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e58b82e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:56,176 DEBUG [Thread-78 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:56,177 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,181 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767996181,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,183 DEBUG [Reader=1,bindAddress=localhost,port=36727 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,184 DEBUG [Reader=1,bindAddress=localhost,port=36727 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,184 DEBUG [Reader=1,bindAddress=localhost,port=36727 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,185 DEBUG [Reader=1,bindAddress=localhost,port=36727 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:56,186 DEBUG [Reader=1,bindAddress=localhost,port=36727 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,186 DEBUG [Reader=1,bindAddress=localhost,port=36727 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,186 DEBUG [Reader=1,bindAddress=localhost,port=36727 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:56,187 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-16T14:39:56,187 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.FailedServers(52): Added failed server with address localhost:36727 to list caused by javax.security.sasl.SaslException: No common protection layer between client and server 2024-11-16T14:39:56,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:56,188 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 36727 2024-11-16T14:39:56,188 INFO [Listener,port=36727 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=36727: stopping 2024-11-16T14:39:56,188 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:56,188 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:56,195 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=58 (was 55) Potentially hanging thread: Relogin-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Reader=1,bindAddress=localhost,port=36727 java.base@17.0.11/sun.nio.ch.SelectorImpl.selectedKeys(SelectorImpl.java:91) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:169) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6618 (was 6636) 2024-11-16T14:39:56,202 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=57, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6618 2024-11-16T14:39:56,216 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:56,216 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:56,220 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767996220,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:56,223 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:56,226 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34237 2024-11-16T14:39:56,227 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:56,227 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:56,227 INFO [Listener,port=34237 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34237: starting 2024-11-16T14:39:56,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c0bc743, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:56,228 DEBUG [Thread-81 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:56,230 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,242 WARN [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731767996221 2024-11-16T14:39:56,243 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767996243,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,246 DEBUG [Reader=1,bindAddress=localhost,port=34237 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,246 DEBUG [Reader=1,bindAddress=localhost,port=34237 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,246 DEBUG [Reader=1,bindAddress=localhost,port=34237 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,247 DEBUG [Reader=1,bindAddress=localhost,port=34237 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:56,248 DEBUG [Reader=1,bindAddress=localhost,port=34237 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,248 DEBUG [Reader=1,bindAddress=localhost,port=34237 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,249 DEBUG [Reader=1,bindAddress=localhost,port=34237 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:56,249 DEBUG [Reader=1,bindAddress=localhost,port=34237 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,249 DEBUG [Reader=1,bindAddress=localhost,port=34237 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,249 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:56,249 DEBUG [Reader=1,bindAddress=localhost,port=34237 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,250 DEBUG [Reader=1,bindAddress=localhost,port=34237 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:56,250 INFO [Reader=1,bindAddress=localhost,port=34237 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:56,250 INFO [Reader=1,bindAddress=localhost,port=34237 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46030, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:56,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:56,257 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34237 2024-11-16T14:39:56,257 INFO [Listener,port=34237 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34237: stopping 2024-11-16T14:39:56,258 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:56,258 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:56,258 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:56,260 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=42665 2024-11-16T14:39:56,260 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:56,260 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:56,260 INFO [Listener,port=42665 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=42665: starting 2024-11-16T14:39:56,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a02c8dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:56,261 DEBUG [Thread-84 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:56,270 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,272 DEBUG [Reader=1,bindAddress=localhost,port=42665 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,273 DEBUG [Reader=1,bindAddress=localhost,port=42665 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,273 DEBUG [Reader=1,bindAddress=localhost,port=42665 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,273 DEBUG [Reader=1,bindAddress=localhost,port=42665 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:56,274 DEBUG [Reader=1,bindAddress=localhost,port=42665 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,274 DEBUG [Reader=1,bindAddress=localhost,port=42665 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,275 DEBUG [Reader=1,bindAddress=localhost,port=42665 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:56,275 DEBUG [Reader=1,bindAddress=localhost,port=42665 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,275 DEBUG [Reader=1,bindAddress=localhost,port=42665 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,275 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:56,275 DEBUG [Reader=1,bindAddress=localhost,port=42665 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,275 DEBUG [Reader=1,bindAddress=localhost,port=42665 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:56,276 INFO [Reader=1,bindAddress=localhost,port=42665 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:56,276 INFO [Reader=1,bindAddress=localhost,port=42665 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42272, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:56,283 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:56,283 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 42665 2024-11-16T14:39:56,283 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:56,283 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:56,284 INFO [Listener,port=42665 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=42665: stopping 2024-11-16T14:39:56,284 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:56,285 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39847 2024-11-16T14:39:56,285 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:56,286 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:56,286 INFO [Listener,port=39847 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39847: starting 2024-11-16T14:39:56,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11c15f92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:56,287 DEBUG [Thread-87 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:56,292 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,295 DEBUG [Reader=1,bindAddress=localhost,port=39847 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,296 DEBUG [Reader=1,bindAddress=localhost,port=39847 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,296 DEBUG [Reader=1,bindAddress=localhost,port=39847 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,297 DEBUG [Reader=1,bindAddress=localhost,port=39847 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:56,298 DEBUG [Reader=1,bindAddress=localhost,port=39847 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,298 DEBUG [Reader=1,bindAddress=localhost,port=39847 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,298 DEBUG [Reader=1,bindAddress=localhost,port=39847 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:56,299 DEBUG [Reader=1,bindAddress=localhost,port=39847 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,299 DEBUG [Reader=1,bindAddress=localhost,port=39847 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,299 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:56,299 DEBUG [Reader=1,bindAddress=localhost,port=39847 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,299 DEBUG [Reader=1,bindAddress=localhost,port=39847 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:56,299 INFO [Reader=1,bindAddress=localhost,port=39847 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:56,300 INFO [Reader=1,bindAddress=localhost,port=39847 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39632, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:56,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:56,308 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39847 2024-11-16T14:39:56,308 INFO [Listener,port=39847 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39847: stopping 2024-11-16T14:39:56,308 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:56,308 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:56,308 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:56,310 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41349 2024-11-16T14:39:56,310 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:56,310 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:56,310 INFO [Listener,port=41349 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41349: starting 2024-11-16T14:39:56,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cb266b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:56,311 DEBUG [Thread-90 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:56,313 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,314 DEBUG [Reader=1,bindAddress=localhost,port=41349 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,315 DEBUG [Reader=1,bindAddress=localhost,port=41349 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,315 DEBUG [Reader=1,bindAddress=localhost,port=41349 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,316 DEBUG [Reader=1,bindAddress=localhost,port=41349 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:56,316 DEBUG [Reader=1,bindAddress=localhost,port=41349 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,316 DEBUG [Reader=1,bindAddress=localhost,port=41349 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,316 DEBUG [Reader=1,bindAddress=localhost,port=41349 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:56,317 DEBUG [Reader=1,bindAddress=localhost,port=41349 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,317 DEBUG [Reader=1,bindAddress=localhost,port=41349 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,317 DEBUG [Reader=1,bindAddress=localhost,port=41349 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,317 DEBUG [Reader=1,bindAddress=localhost,port=41349 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int 2024-11-16T14:39:56,317 INFO [Reader=1,bindAddress=localhost,port=41349 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:56,317 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:56,318 INFO [Reader=1,bindAddress=localhost,port=41349 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51544, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:56,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:56,327 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41349 2024-11-16T14:39:56,327 INFO [Listener,port=41349 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41349: stopping 2024-11-16T14:39:56,327 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:56,327 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:56,327 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:56,328 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40233 2024-11-16T14:39:56,330 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:56,331 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:56,331 INFO [Listener,port=40233 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40233: starting 2024-11-16T14:39:56,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24e63b4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:56,331 DEBUG [Thread-93 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:56,333 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,334 DEBUG [Reader=1,bindAddress=localhost,port=40233 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,335 DEBUG [Reader=1,bindAddress=localhost,port=40233 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,335 DEBUG [Reader=1,bindAddress=localhost,port=40233 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,336 DEBUG [Reader=1,bindAddress=localhost,port=40233 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:56,336 DEBUG [Reader=1,bindAddress=localhost,port=40233 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,336 DEBUG [Reader=1,bindAddress=localhost,port=40233 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,336 DEBUG [Reader=1,bindAddress=localhost,port=40233 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:56,337 DEBUG [Reader=1,bindAddress=localhost,port=40233 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,337 DEBUG [Reader=1,bindAddress=localhost,port=40233 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,337 DEBUG [Reader=1,bindAddress=localhost,port=40233 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,337 DEBUG [Reader=1,bindAddress=localhost,port=40233 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-16T14:39:56,337 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:56,337 INFO [Reader=1,bindAddress=localhost,port=40233 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:56,338 INFO [Reader=1,bindAddress=localhost,port=40233 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40984, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:56,346 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:56,346 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40233 2024-11-16T14:39:56,346 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:56,346 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:56,347 INFO [Listener,port=40233 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40233: stopping 2024-11-16T14:39:56,353 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=65 (was 57) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: FifoRpcScheduler.handler-pool-0 java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Listener,port=40233 java.base@17.0.11/sun.nio.ch.EPoll.ctl(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implDereg(EPollSelectorImpl.java:230) java.base@17.0.11/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:267) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:134) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:213) Potentially hanging thread: RpcServer idle connection scanner for port 40233 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6585 (was 6618) 2024-11-16T14:39:56,359 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=62, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6585 2024-11-16T14:39:56,368 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:56,368 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:56,370 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767996370,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:56,372 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:56,373 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34071 2024-11-16T14:39:56,373 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:56,373 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:56,373 INFO [Listener,port=34071 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34071: starting 2024-11-16T14:39:56,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2902dc7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:56,374 DEBUG [Thread-96 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T14:39:56,379 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(424): process preamble call response with response type null 2024-11-16T14:39:56,380 WARN [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call 2024-11-16T14:39:56,380 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes 2024-11-16T14:39:56,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:56,380 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34071 2024-11-16T14:39:56,381 INFO [Listener,port=34071 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34071: stopping 2024-11-16T14:39:56,381 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:56,381 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:56,386 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=64 (was 62) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.lang.String.(String.java:521) java.base@17.0.11/java.lang.String.(String.java:1417) app//org.apache.maven.surefire.api.report.ConsoleOutputCapture$ForwardingPrintStream.write(ConsoleOutputCapture.java:56) app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:250) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:283) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:294) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:217) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6584 (was 6585) 2024-11-16T14:39:56,392 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=63, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6584 2024-11-16T14:39:56,401 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:56,401 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:56,403 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767996403,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:56,405 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:56,406 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45035 2024-11-16T14:39:56,407 WARN [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! ********* 2024-11-16T14:39:56,407 WARN [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients 2024-11-16T14:39:56,407 WARN [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true). 2024-11-16T14:39:56,407 WARN [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user 2024-11-16T14:39:56,407 WARN [Time-limited test {}] ipc.RpcServer(380): impersonation is possible! 2024-11-16T14:39:56,407 WARN [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible, 2024-11-16T14:39:56,407 WARN [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml 2024-11-16T14:39:56,407 WARN [Time-limited test {}] ipc.RpcServer(383): **************************** 2024-11-16T14:39:56,407 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:56,409 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:56,409 INFO [Listener,port=45035 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45035: starting 2024-11-16T14:39:56,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57ce90a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:56,410 DEBUG [Thread-99 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T14:39:56,412 WARN [Reader=1,bindAddress=localhost,port=45035 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1 2024-11-16T14:39:56,412 INFO [Reader=1,bindAddress=localhost,port=45035 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40872, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T14:39:56,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:56,417 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45035 2024-11-16T14:39:56,417 INFO [Listener,port=45035 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45035: stopping 2024-11-16T14:39:56,419 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:56,419 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:56,423 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=68 (was 63) Potentially hanging thread: Listener,port=45035 java.base@17.0.11/sun.nio.ch.FileDispatcherImpl.closeIntFD(Native Method) java.base@17.0.11/sun.nio.ch.EventFD.close(EventFD.java:56) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implClose(EPollSelectorImpl.java:219) java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:185) java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:256) Potentially hanging thread: Reader=1,bindAddress=localhost,port=45035 java.base@17.0.11/sun.nio.ch.FileDispatcherImpl.close0(Native Method) java.base@17.0.11/sun.nio.ch.SocketDispatcher.close(SocketDispatcher.java:70) java.base@17.0.11/sun.nio.ch.SocketChannelImpl.tryClose(SocketChannelImpl.java:980) java.base@17.0.11/sun.nio.ch.SocketChannelImpl.tryFinishClose(SocketChannelImpl.java:994) java.base@17.0.11/sun.nio.ch.SocketChannelImpl.kill(SocketChannelImpl.java:1088) java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:194) java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:151) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer idle connection scanner for port 45035 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RpcServer.responder java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:115) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6583 (was 6584) 2024-11-16T14:39:56,428 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=64, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6582 2024-11-16T14:39:56,437 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:56,438 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:56,440 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767996440,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:56,442 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:56,443 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40103 2024-11-16T14:39:56,443 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:56,443 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:56,443 INFO [Listener,port=40103 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40103: starting 2024-11-16T14:39:56,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f3a7f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:56,444 DEBUG [Thread-102 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:56,446 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,448 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767996448,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,450 DEBUG [Reader=1,bindAddress=localhost,port=40103 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,451 DEBUG [Reader=1,bindAddress=localhost,port=40103 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,451 DEBUG [Reader=1,bindAddress=localhost,port=40103 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,452 DEBUG [Reader=1,bindAddress=localhost,port=40103 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:56,452 DEBUG [Reader=1,bindAddress=localhost,port=40103 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,452 DEBUG [Reader=1,bindAddress=localhost,port=40103 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,452 DEBUG [Reader=1,bindAddress=localhost,port=40103 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:56,453 DEBUG [Reader=1,bindAddress=localhost,port=40103 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,453 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:56,453 DEBUG [Reader=1,bindAddress=localhost,port=40103 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,453 DEBUG [Reader=1,bindAddress=localhost,port=40103 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,453 DEBUG [Reader=1,bindAddress=localhost,port=40103 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:56,453 INFO [Reader=1,bindAddress=localhost,port=40103 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:56,454 INFO [Reader=1,bindAddress=localhost,port=40103 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56024, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:56,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:56,460 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40103 2024-11-16T14:39:56,460 INFO [Listener,port=40103 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40103: stopping 2024-11-16T14:39:56,461 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:56,461 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:56,467 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=66 (was 64) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.fetchStackFrames(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.fetchStackFrames(StackStreamFactory.java:386) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.getNextBatch(StackStreamFactory.java:322) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.peekFrame(StackStreamFactory.java:263) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.hasNext(StackStreamFactory.java:351) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.tryAdvance(StackStreamFactory.java:593) java.base@17.0.11/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129) java.base@17.0.11/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527) java.base@17.0.11/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513) java.base@17.0.11/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499) java.base@17.0.11/java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150) java.base@17.0.11/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) java.base@17.0.11/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647) app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$15(StackLocator.java:100) app//org.apache.logging.log4j.util.StackLocator$$Lambda$125/0x00007fe1f418c260.apply(Unknown Source) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243) java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499) app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97) app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121) app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6579 (was 6582) 2024-11-16T14:39:56,473 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=65, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6579 2024-11-16T14:39:56,481 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:56,481 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:56,483 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767996483,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:56,485 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:56,486 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=43631 2024-11-16T14:39:56,486 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:56,486 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:56,486 INFO [Listener,port=43631 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=43631: starting 2024-11-16T14:39:56,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39271f19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:56,487 DEBUG [Thread-105 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:56,488 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,492 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767996492,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,493 DEBUG [Reader=1,bindAddress=localhost,port=43631 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,494 DEBUG [Reader=1,bindAddress=localhost,port=43631 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,494 DEBUG [Reader=1,bindAddress=localhost,port=43631 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,495 DEBUG [Reader=1,bindAddress=localhost,port=43631 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:56,495 DEBUG [Reader=1,bindAddress=localhost,port=43631 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,495 DEBUG [Reader=1,bindAddress=localhost,port=43631 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,496 DEBUG [Reader=1,bindAddress=localhost,port=43631 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:56,496 DEBUG [Reader=1,bindAddress=localhost,port=43631 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,496 DEBUG [Reader=1,bindAddress=localhost,port=43631 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,496 DEBUG [Reader=1,bindAddress=localhost,port=43631 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,496 DEBUG [Reader=1,bindAddress=localhost,port=43631 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-16T14:39:56,496 INFO [Reader=1,bindAddress=localhost,port=43631 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:56,496 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:56,497 INFO [Reader=1,bindAddress=localhost,port=43631 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44738, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:56,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:56,507 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 43631 2024-11-16T14:39:56,507 INFO [Listener,port=43631 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=43631: stopping 2024-11-16T14:39:56,507 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:56,507 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:56,507 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:56,509 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=38911 2024-11-16T14:39:56,509 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:56,509 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:56,509 INFO [Listener,port=38911 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=38911: starting 2024-11-16T14:39:56,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b21be99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:56,510 DEBUG [Thread-108 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:56,511 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,513 DEBUG [Reader=1,bindAddress=localhost,port=38911 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,513 DEBUG [Reader=1,bindAddress=localhost,port=38911 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,513 DEBUG [Reader=1,bindAddress=localhost,port=38911 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,514 DEBUG [Reader=1,bindAddress=localhost,port=38911 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:56,515 DEBUG [Reader=1,bindAddress=localhost,port=38911 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,515 DEBUG [Reader=1,bindAddress=localhost,port=38911 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,515 DEBUG [Reader=1,bindAddress=localhost,port=38911 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:56,515 DEBUG [Reader=1,bindAddress=localhost,port=38911 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:56,515 DEBUG [Reader=1,bindAddress=localhost,port=38911 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:56,515 DEBUG [Reader=1,bindAddress=localhost,port=38911 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:56,515 DEBUG [Reader=1,bindAddress=localhost,port=38911 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-16T14:39:56,516 INFO [Reader=1,bindAddress=localhost,port=38911 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:56,516 INFO [Reader=1,bindAddress=localhost,port=38911 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50282, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:58,518 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-16T14:39:58,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:58,518 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.FailedServers(52): Added failed server with address localhost:38911 to list caused by java.io.IOException: org.apache.hbase.thirdparty.io.netty.handler.timeout.ReadTimeoutException 2024-11-16T14:39:58,518 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 38911 2024-11-16T14:39:58,519 INFO [Listener,port=38911 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=38911: stopping 2024-11-16T14:39:58,519 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:58,519 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:58,529 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=68 (was 65) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.afterLogEvent(AwaitCompletionReliabilityStrategy.java:127) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:84) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6468 (was 6579) 2024-11-16T14:39:58,536 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=67, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6467 2024-11-16T14:39:58,547 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:58,547 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:58,550 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767998549,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:58,552 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:58,553 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=37625 2024-11-16T14:39:58,553 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:58,553 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:58,553 INFO [Listener,port=37625 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=37625: starting 2024-11-16T14:39:58,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4960f39c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:58,554 DEBUG [Thread-111 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:58,556 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,558 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767998558,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,559 DEBUG [Reader=1,bindAddress=localhost,port=37625 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,560 DEBUG [Reader=1,bindAddress=localhost,port=37625 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:58,560 DEBUG [Reader=1,bindAddress=localhost,port=37625 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,560 DEBUG [Reader=1,bindAddress=localhost,port=37625 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:58,561 DEBUG [Reader=1,bindAddress=localhost,port=37625 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:58,561 DEBUG [Reader=1,bindAddress=localhost,port=37625 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,561 DEBUG [Reader=1,bindAddress=localhost,port=37625 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:58,561 DEBUG [Reader=1,bindAddress=localhost,port=37625 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:58,562 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:58,562 DEBUG [Reader=1,bindAddress=localhost,port=37625 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,562 DEBUG [Reader=1,bindAddress=localhost,port=37625 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,562 DEBUG [Reader=1,bindAddress=localhost,port=37625 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:58,562 INFO [Reader=1,bindAddress=localhost,port=37625 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:58,562 INFO [Reader=1,bindAddress=localhost,port=37625 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36856, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:58,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:58,568 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 37625 2024-11-16T14:39:58,568 INFO [Listener,port=37625 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=37625: stopping 2024-11-16T14:39:58,568 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:58,568 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:58,573 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=68 (was 67) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6466 (was 6467) 2024-11-16T14:39:58,577 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=68, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6466 2024-11-16T14:39:58,587 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:58,587 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:58,589 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767998589,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:58,591 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:58,592 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=37813 2024-11-16T14:39:58,593 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:58,593 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:58,593 INFO [Listener,port=37813 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=37813: starting 2024-11-16T14:39:58,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fd72b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:58,593 DEBUG [Thread-114 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:58,595 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,597 WARN [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731767998590 2024-11-16T14:39:58,597 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767998597,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,598 DEBUG [Reader=1,bindAddress=localhost,port=37813 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,599 DEBUG [Reader=1,bindAddress=localhost,port=37813 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:58,599 DEBUG [Reader=1,bindAddress=localhost,port=37813 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,600 DEBUG [Reader=1,bindAddress=localhost,port=37813 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-16T14:39:58,601 DEBUG [Reader=1,bindAddress=localhost,port=37813 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:58,601 DEBUG [Reader=1,bindAddress=localhost,port=37813 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,601 DEBUG [Reader=1,bindAddress=localhost,port=37813 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-16T14:39:58,602 DEBUG [Reader=1,bindAddress=localhost,port=37813 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-16T14:39:58,602 DEBUG [Reader=1,bindAddress=localhost,port=37813 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,602 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:58,602 DEBUG [Reader=1,bindAddress=localhost,port=37813 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,602 DEBUG [Reader=1,bindAddress=localhost,port=37813 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:58,602 INFO [Reader=1,bindAddress=localhost,port=37813 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:58,603 INFO [Reader=1,bindAddress=localhost,port=37813 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50416, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:58,610 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:58,610 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 37813 2024-11-16T14:39:58,610 INFO [Listener,port=37813 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=37813: stopping 2024-11-16T14:39:58,610 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:58,612 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:58,617 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=70 (was 68) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.lang.ThreadLocal.get(ThreadLocal.java:163) app//org.apache.logging.log4j.spi.AbstractLogger.getRecursionDepthHolder(AbstractLogger.java:2166) app//org.apache.logging.log4j.spi.AbstractLogger.getRecursionDepth(AbstractLogger.java:2192) app//org.apache.logging.log4j.core.layout.AbstractStringLayout.getStringBuilder(AbstractStringLayout.java:129) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:225) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6463 (was 6466) 2024-11-16T14:39:58,622 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=69, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6463 2024-11-16T14:39:58,630 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:58,630 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:58,632 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767998631,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:58,633 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:58,634 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=35115 2024-11-16T14:39:58,634 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:58,634 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-16T14:39:58,634 INFO [Listener,port=35115 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=35115: starting 2024-11-16T14:39:58,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10a405d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:58,635 DEBUG [Thread-117 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:58,637 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,639 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767998639,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,640 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:58,641 INFO [Reader=1,bindAddress=localhost,port=35115 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T14:39:58,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:58,646 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 35115 2024-11-16T14:39:58,646 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-16T14:39:58,646 INFO [Listener,port=35115 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=35115: stopping 2024-11-16T14:39:58,646 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-16T14:39:58,652 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=70 (was 69) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6462 (was 6463) 2024-11-16T14:39:58,659 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=70, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6462 2024-11-16T14:39:58,671 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:58,671 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:58,673 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767998673,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:58,674 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:58,675 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:58,676 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37401 2024-11-16T14:39:58,676 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:58,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478f2ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:58,677 DEBUG [Thread-118 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:58,678 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,680 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767998680,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,682 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,682 DEBUG [NettyRpcServer-19-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,684 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:58,684 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,685 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:58,685 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,685 DEBUG [NettyRpcServer-19-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,686 DEBUG [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-16T14:39:58,686 INFO [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:58,687 INFO [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49914, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:58,688 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:58,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:58,697 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37401 2024-11-16T14:39:58,703 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=72 (was 70) Potentially hanging thread: NettyRpcServer-19-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-19-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=475 (was 466) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6459 (was 6462) 2024-11-16T14:39:58,709 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=72, OpenFileDescriptor=475, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6459 2024-11-16T14:39:58,718 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:58,718 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:58,720 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767998720,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:58,722 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:58,722 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:58,723 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39117 2024-11-16T14:39:58,723 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:58,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b691a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-16T14:39:58,724 DEBUG [Thread-119 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:58,725 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,727 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767998727,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,729 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-16T14:39:58,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:58,729 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.FailedServers(52): Added failed server with address localhost:39117 to list caused by org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections. 2024-11-16T14:39:58,729 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39117 2024-11-16T14:39:58,735 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=74 (was 72) Potentially hanging thread: NettyRpcServer-20-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-20-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=484 (was 475) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6455 (was 6459) 2024-11-16T14:39:58,741 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=74, OpenFileDescriptor=484, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6455 2024-11-16T14:39:58,752 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:58,752 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:58,755 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767998754,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:58,756 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:58,757 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:58,757 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33395 2024-11-16T14:39:58,757 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:58,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73fe48bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:58,758 DEBUG [Thread-120 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:58,759 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,762 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767998762,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,764 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,764 DEBUG [NettyRpcServer-21-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,765 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:58,766 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,766 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:58,767 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-16T14:39:58,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:58,767 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.FailedServers(52): Added failed server with address localhost:33395 to list caused by javax.security.sasl.SaslException: No common protection layer between client and server 2024-11-16T14:39:58,767 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33395 2024-11-16T14:39:58,774 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=76 (was 74) Potentially hanging thread: NettyRpcServer-21-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-21-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=493 (was 484) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6453 (was 6455) 2024-11-16T14:39:58,781 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=76, OpenFileDescriptor=493, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6452 2024-11-16T14:39:58,790 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:58,791 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:58,793 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767998792,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:58,794 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:58,795 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:58,796 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43991 2024-11-16T14:39:58,796 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:58,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54975df9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:58,797 DEBUG [Thread-121 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:58,798 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,801 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767998801,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,803 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,803 DEBUG [NettyRpcServer-22-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,805 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:58,805 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,805 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:58,806 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,806 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:58,806 DEBUG [NettyRpcServer-22-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,806 DEBUG [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:58,806 INFO [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:58,807 INFO [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44496, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:58,814 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:58,815 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43991 2024-11-16T14:39:58,815 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:58,816 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:58,817 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33829 2024-11-16T14:39:58,817 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:58,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55043756, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:58,838 WARN [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731767998793 2024-11-16T14:39:58,843 DEBUG [Thread-122 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:58,844 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,846 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,846 DEBUG [NettyRpcServer-23-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,848 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:58,849 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,849 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:58,849 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,850 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:58,850 DEBUG [NettyRpcServer-23-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,850 DEBUG [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:58,850 INFO [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:58,850 INFO [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41102, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:58,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:58,859 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33829 2024-11-16T14:39:58,860 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:58,860 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:58,861 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45201 2024-11-16T14:39:58,861 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:58,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@136b3045, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:58,863 DEBUG [Thread-123 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:58,864 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,866 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,866 DEBUG [NettyRpcServer-24-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,868 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:58,869 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,869 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:58,870 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,870 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:58,870 DEBUG [NettyRpcServer-24-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,870 DEBUG [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:58,870 INFO [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:58,871 INFO [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44216, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:58,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:58,879 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45201 2024-11-16T14:39:58,880 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:58,880 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:58,883 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42337 2024-11-16T14:39:58,883 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:58,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@284b66d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:58,884 DEBUG [Thread-124 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:58,885 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,887 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,887 DEBUG [NettyRpcServer-25-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,888 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:58,889 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,889 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:58,889 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,890 DEBUG [NettyRpcServer-25-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,890 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:58,890 DEBUG [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int 2024-11-16T14:39:58,890 INFO [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:58,890 INFO [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41166, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:58,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:58,898 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42337 2024-11-16T14:39:58,899 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:58,899 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:58,900 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40753 2024-11-16T14:39:58,901 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:58,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681d3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:58,902 DEBUG [Thread-125 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:58,903 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,904 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,905 DEBUG [NettyRpcServer-26-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,906 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:58,906 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,907 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:58,907 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:58,907 DEBUG [NettyRpcServer-26-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:58,907 DEBUG [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-16T14:39:58,907 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:58,907 INFO [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:58,908 INFO [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33810, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:58,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:58,917 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40753 2024-11-16T14:39:58,923 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=86 (was 76) Potentially hanging thread: NettyRpcServer-22-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-24-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-23-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-26-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-25-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-25-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-23-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-22-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-24-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-26-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=538 (was 493) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6439 (was 6452) 2024-11-16T14:39:58,929 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=86, OpenFileDescriptor=538, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6438 2024-11-16T14:39:58,939 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:58,939 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:58,941 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767998941,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:58,943 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:58,943 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:58,944 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40873 2024-11-16T14:39:58,944 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:58,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74e36d02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:58,945 DEBUG [Thread-126 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T14:39:58,947 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(424): process preamble call response with response type null 2024-11-16T14:39:58,947 WARN [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call 2024-11-16T14:39:58,947 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes 2024-11-16T14:39:58,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:58,947 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40873 2024-11-16T14:39:58,953 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=88 (was 86) Potentially hanging thread: NettyRpcServer-27-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-27-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=547 (was 538) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6435 (was 6438) 2024-11-16T14:39:58,960 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=88, OpenFileDescriptor=547, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6435 2024-11-16T14:39:58,971 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:58,971 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:58,973 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767998973,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:58,975 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:58,975 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:58,976 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33189 2024-11-16T14:39:58,976 WARN [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! ********* 2024-11-16T14:39:58,976 WARN [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients 2024-11-16T14:39:58,976 WARN [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true). 2024-11-16T14:39:58,976 WARN [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user 2024-11-16T14:39:58,976 WARN [Time-limited test {}] ipc.RpcServer(380): impersonation is possible! 2024-11-16T14:39:58,976 WARN [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible, 2024-11-16T14:39:58,976 WARN [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml 2024-11-16T14:39:58,976 WARN [Time-limited test {}] ipc.RpcServer(383): **************************** 2024-11-16T14:39:58,976 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:58,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d602104, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:58,977 DEBUG [Thread-127 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T14:39:58,979 WARN [NettyRpcServer-28-2 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1 2024-11-16T14:39:58,979 INFO [NettyRpcServer-28-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T14:39:58,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:58,985 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33189 2024-11-16T14:39:58,991 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=90 (was 88) Potentially hanging thread: NettyRpcServer-28-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-28-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=556 (was 547) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6433 (was 6435) 2024-11-16T14:39:58,998 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=90, OpenFileDescriptor=556, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6432 2024-11-16T14:39:59,011 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:59,011 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:59,013 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767999013,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:59,015 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:59,015 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:59,016 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39771 2024-11-16T14:39:59,016 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:59,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c064fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:59,017 DEBUG [Thread-128 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:59,018 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:59,020 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767999020,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:59,022 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=498 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:59,022 DEBUG [NettyRpcServer-29-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:59,023 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 107 from saslServer. 2024-11-16T14:39:59,024 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:59,024 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:59,024 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:59,024 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:59,024 DEBUG [NettyRpcServer-29-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:59,025 DEBUG [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:39:59,025 INFO [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:59,025 INFO [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59064, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:59,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:59,032 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39771 2024-11-16T14:39:59,040 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=92 (was 90) Potentially hanging thread: NettyRpcServer-29-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-29-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=565 (was 556) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6429 (was 6432) 2024-11-16T14:39:59,046 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=92, OpenFileDescriptor=565, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6429 2024-11-16T14:39:59,058 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:39:59,058 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:39:59,060 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731767999060,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:39:59,062 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:59,062 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:59,064 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42515 2024-11-16T14:39:59,064 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:59,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e38393c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:59,065 DEBUG [Thread-129 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:59,066 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:59,069 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731767999069,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:59,070 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:59,070 DEBUG [NettyRpcServer-30-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:59,071 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:59,072 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:59,072 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:59,073 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:59,073 DEBUG [NettyRpcServer-30-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:59,073 DEBUG [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-16T14:39:59,073 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:39:59,073 INFO [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:59,074 INFO [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57448, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:39:59,083 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:39:59,083 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42515 2024-11-16T14:39:59,084 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:39:59,084 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:39:59,085 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34273 2024-11-16T14:39:59,085 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:39:59,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5586da8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:39:59,086 DEBUG [Thread-130 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:39:59,087 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:59,088 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:59,088 DEBUG [NettyRpcServer-31-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:59,090 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:39:59,090 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:59,090 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:39:59,091 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:39:59,091 DEBUG [NettyRpcServer-31-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:39:59,091 DEBUG [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-16T14:39:59,091 INFO [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:39:59,092 INFO [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35052, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:40:01,094 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-16T14:40:01,094 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.FailedServers(52): Added failed server with address localhost:34273 to list caused by java.io.IOException: org.apache.hbase.thirdparty.io.netty.handler.timeout.ReadTimeoutException 2024-11-16T14:40:01,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:40:01,095 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34273 2024-11-16T14:40:01,105 WARN [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731767999061 2024-11-16T14:40:01,114 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=96 (was 92) Potentially hanging thread: NettyRpcServer-31-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-30-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-31-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-30-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=583 (was 565) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=375 (was 304) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6360 (was 6429) 2024-11-16T14:40:01,128 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=96, OpenFileDescriptor=583, MaxFileDescriptor=1048576, SystemLoadAverage=375, ProcessCount=11, AvailableMemoryMB=6360 2024-11-16T14:40:01,140 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:40:01,140 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:40:01,146 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731768001146,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:40:01,149 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:40:01,149 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:40:01,150 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41155 2024-11-16T14:40:01,150 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:40:01,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b913045, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:40:01,169 DEBUG [Thread-131 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:40:01,170 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:40:01,174 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731768001174,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:40:01,185 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:40:01,185 DEBUG [NettyRpcServer-32-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:40:01,187 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:40:01,190 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:40:01,190 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:40:01,191 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:40:01,192 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:40:01,193 DEBUG [NettyRpcServer-32-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:40:01,193 DEBUG [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:40:01,193 INFO [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:40:01,193 INFO [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37544, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:40:01,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:40:01,220 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41155 2024-11-16T14:40:01,242 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=99 (was 96) Potentially hanging thread: NettyRpcServer-32-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-32-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: FifoRpcScheduler.handler-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=592 (was 583) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=375 (was 375), ProcessCount=11 (was 11), AvailableMemoryMB=6352 (was 6360) 2024-11-16T14:40:01,251 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=98, OpenFileDescriptor=592, MaxFileDescriptor=1048576, SystemLoadAverage=375, ProcessCount=11, AvailableMemoryMB=6351 2024-11-16T14:40:01,265 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:40:01,265 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:40:01,270 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731768001269,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:40:01,273 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:40:01,273 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:40:01,287 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44689 2024-11-16T14:40:01,287 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:40:01,288 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec57c5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:40:01,311 DEBUG [Thread-132 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:40:01,312 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:40:01,316 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731768001315,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:40:01,319 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-16T14:40:01,320 DEBUG [NettyRpcServer-33-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-16T14:40:01,322 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-16T14:40:01,324 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-16T14:40:01,324 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-16T14:40:01,325 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-16T14:40:01,325 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:40:01,325 DEBUG [NettyRpcServer-33-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-16T14:40:01,325 DEBUG [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-16T14:40:01,326 INFO [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-16T14:40:01,326 INFO [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51720, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-16T14:40:01,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:40:01,343 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44689 2024-11-16T14:40:01,368 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=100 (was 98) Potentially hanging thread: NettyRpcServer-33-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-33-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=601 (was 592) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=375 (was 375), ProcessCount=11 (was 11), AvailableMemoryMB=6345 (was 6351) 2024-11-16T14:40:01,384 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=100, OpenFileDescriptor=601, MaxFileDescriptor=1048576, SystemLoadAverage=375, ProcessCount=11, AvailableMemoryMB=6345 2024-11-16T14:40:01,400 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-16T14:40:01,401 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-16T14:40:01,404 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731768001403,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-16T14:40:01,417 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-16T14:40:01,418 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:40:01,427 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46189 2024-11-16T14:40:01,427 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T14:40:01,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18b1282e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:40:01,432 DEBUG [Thread-133 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-16T14:40:01,435 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-16T14:40:01,440 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731768001440,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-16T14:40:01,456 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-16T14:40:01,457 INFO [NettyRpcServer-34-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T14:40:01,469 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:40:01,470 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46189 2024-11-16T14:40:01,484 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=102 (was 100) Potentially hanging thread: NettyRpcServer-34-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-34-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=610 (was 601) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=375 (was 375), ProcessCount=11 (was 11), AvailableMemoryMB=6341 (was 6345) 2024-11-16T14:40:01,485 INFO [Time-limited test {}] impl.DefaultInternalKdcServerImpl(102): Default Internal kdc server stopped.