2024-11-16 04:03:28,121 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@14a50707 2024-11-16 04:03:28,132 main DEBUG Took 0.009572 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 04:03:28,133 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 04:03:28,133 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 04:03:28,134 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 04:03:28,135 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:03:28,145 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 04:03:28,156 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:03:28,158 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:03:28,158 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:03:28,159 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:03:28,159 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:03:28,160 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:03:28,160 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:03:28,161 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:03:28,161 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:03:28,161 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:03:28,162 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:03:28,162 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:03:28,163 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:03:28,163 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:03:28,164 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:03:28,164 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:03:28,164 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:03:28,165 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:03:28,165 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:03:28,165 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:03:28,166 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:03:28,166 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:03:28,166 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:03:28,167 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:03:28,167 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:03:28,167 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 04:03:28,169 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:03:28,170 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 04:03:28,172 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 04:03:28,172 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 04:03:28,173 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 04:03:28,174 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 04:03:28,181 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 04:03:28,184 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 04:03:28,185 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 04:03:28,185 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 04:03:28,186 main DEBUG createAppenders(={Console}) 2024-11-16 04:03:28,186 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@14a50707 initialized 2024-11-16 04:03:28,187 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@14a50707 2024-11-16 04:03:28,187 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@14a50707 OK. 2024-11-16 04:03:28,187 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 04:03:28,188 main DEBUG OutputStream closed 2024-11-16 04:03:28,188 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 04:03:28,188 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 04:03:28,189 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55cb6996 OK 2024-11-16 04:03:28,257 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 04:03:28,259 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 04:03:28,260 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 04:03:28,261 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 04:03:28,262 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 04:03:28,263 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 04:03:28,263 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 04:03:28,263 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 04:03:28,264 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 04:03:28,264 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 04:03:28,265 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 04:03:28,265 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 04:03:28,265 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 04:03:28,266 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 04:03:28,266 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 04:03:28,266 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 04:03:28,267 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 04:03:28,268 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 04:03:28,270 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 04:03:28,271 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@28e8dde3) with optional ClassLoader: null 2024-11-16 04:03:28,271 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 04:03:28,272 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@28e8dde3] started OK. 2024-11-16T04:03:28,286 DEBUG [main {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16 04:03:28,288 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 04:03:28,289 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T04:03:28,400 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.ipc.TestNettyTlsIPC timeout: 312 mins 2024-11-16T04:03:28,695 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T04:03:28,774 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=12, OpenFileDescriptor=296, MaxFileDescriptor=1048576, SystemLoadAverage=114, ProcessCount=11, AvailableMemoryMB=7580 2024-11-16T04:03:30,031 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T04:03:30,069 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T04:03:30,088 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T04:03:30,154 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:30,182 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 104621 (auto-detected) 2024-11-16T04:03:30,184 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-16T04:03:30,204 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45609 2024-11-16T04:03:30,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53780854, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:30,239 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:30,493 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:30,526 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:30,526 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:45609 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:03:30,527 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45609 2024-11-16T04:03:30,536 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=18 (was 12) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestNettyTlsIPC-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestNettyTlsIPC-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=296 (was 296), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=136 (was 114) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7464 (was 7580) 2024-11-16T04:03:30,543 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=18, OpenFileDescriptor=296, MaxFileDescriptor=1048576, SystemLoadAverage=136, ProcessCount=11, AvailableMemoryMB=7464 2024-11-16T04:03:30,545 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:30,546 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35557 2024-11-16T04:03:30,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2669441b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:30,547 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:30,549 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:30,557 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:30,638 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:30,721 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:30,813 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:30,841 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:35557 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=289ms, rpcTimeout=100ms 2024-11-16T04:03:30,844 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:30,901 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x8d37106c, L:/127.0.0.1:35557 - R:/127.0.0.1:53964] 2024-11-16T04:03:30,946 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:35557 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=103ms, rpcTimeout=100ms 2024-11-16T04:03:31,029 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:31,097 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:35557 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=150ms, rpcTimeout=100ms 2024-11-16T04:03:31,206 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:35557 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:31,316 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:35557 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:31,427 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:35557 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:31,537 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:35557 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:03:31,647 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:35557 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:03:31,756 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:35557 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:03:31,866 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:35557 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:31,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:31,867 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35557 2024-11-16T04:03:31,880 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=22 (was 18) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: FifoRpcScheduler.handler-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.Threads.sleepWithoutInterrupt(Threads.java:142) app//org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.pause(TestProtobufRpcServiceImpl.java:112) app//org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:155) app//org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) app//org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) app//org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestNettyTlsIPC-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=296 (was 296), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=136 (was 136), ProcessCount=11 (was 11), AvailableMemoryMB=7443 (was 7464) 2024-11-16T04:03:31,885 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=22, OpenFileDescriptor=296, MaxFileDescriptor=1048576, SystemLoadAverage=136, ProcessCount=11, AvailableMemoryMB=7443 2024-11-16T04:03:31,888 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:31,888 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34021 2024-11-16T04:03:31,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f20af72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:31,889 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:31,890 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:31,891 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:31,917 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:31,918 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:31,932 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:31,945 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:31,945 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xa701d279, L:/127.0.0.1:34021 - R:/127.0.0.1:41350] 2024-11-16T04:03:31,980 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:31,983 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:31,991 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:41350 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:32,005 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:32,033 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:32,033 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:32,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:32,041 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=23 (was 22) - Thread LEAK? -, OpenFileDescriptor=298 (was 296) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=136 (was 136), ProcessCount=11 (was 11), AvailableMemoryMB=7442 (was 7443) 2024-11-16T04:03:32,046 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=23, OpenFileDescriptor=297, MaxFileDescriptor=1048576, SystemLoadAverage=136, ProcessCount=11, AvailableMemoryMB=7442 2024-11-16T04:03:32,058 WARN [TestNettyTlsIPC-1-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:32,058 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53964 deadline: 1731729811199 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53964 2024-11-16T04:03:32,064 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53964 deadline: 1731729811308 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53964 2024-11-16T04:03:32,065 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53964 deadline: 1731729811421 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53964 2024-11-16T04:03:32,065 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53964 deadline: 1731729811532 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53964 2024-11-16T04:03:32,066 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53964 deadline: 1731729811641 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53964 2024-11-16T04:03:32,066 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53964 deadline: 1731729811751 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53964 2024-11-16T04:03:32,066 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53964 deadline: 1731729811857 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53964 2024-11-16T04:03:32,103 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:32,103 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34707 2024-11-16T04:03:32,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c6afa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:32,109 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:32,113 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:32,113 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:32,140 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:32,140 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:32,154 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:32,167 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:32,168 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x9e06b0b3, L:/127.0.0.1:34707 - R:/127.0.0.1:51758] 2024-11-16T04:03:32,201 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:32,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:32,229 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34707 2024-11-16T04:03:32,235 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=22 (was 23), OpenFileDescriptor=297 (was 297), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=136 (was 136), ProcessCount=11 (was 11), AvailableMemoryMB=7439 (was 7442) 2024-11-16T04:03:32,240 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=22, OpenFileDescriptor=297, MaxFileDescriptor=1048576, SystemLoadAverage=136, ProcessCount=11, AvailableMemoryMB=7439 2024-11-16T04:03:32,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@305f27d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:32,242 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:32,242 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45851 2024-11-16T04:03:32,243 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:32,245 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:32,246 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:32,274 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:32,274 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:32,288 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:32,300 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:32,301 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xa0e71a4c, L:/127.0.0.1:45851 - R:/127.0.0.1:53786] 2024-11-16T04:03:32,338 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:32,339 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:32,340 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:53786 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:32,341 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:32,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:32,404 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45851 2024-11-16T04:03:32,421 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=23 (was 22) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=297 (was 297), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=136 (was 136), ProcessCount=11 (was 11), AvailableMemoryMB=7448 (was 7439) - AvailableMemoryMB LEAK? - 2024-11-16T04:03:32,425 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=23, OpenFileDescriptor=297, MaxFileDescriptor=1048576, SystemLoadAverage=136, ProcessCount=11, AvailableMemoryMB=7448 2024-11-16T04:03:32,427 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:32,428 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43415 2024-11-16T04:03:32,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e9edc22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:32,428 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:32,429 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:32,430 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:32,457 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:32,457 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:32,472 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:32,485 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:32,486 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x340f3da8, L:/127.0.0.1:43415 - R:/127.0.0.1:55318] 2024-11-16T04:03:32,522 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:32,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:32,531 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43415 2024-11-16T04:03:32,536 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=23 (was 23), OpenFileDescriptor=297 (was 297), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=136 (was 136), ProcessCount=11 (was 11), AvailableMemoryMB=7444 (was 7448) 2024-11-16T04:03:32,540 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=23, OpenFileDescriptor=297, MaxFileDescriptor=1048576, SystemLoadAverage=136, ProcessCount=11, AvailableMemoryMB=7444 2024-11-16T04:03:32,542 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:32,543 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35993 2024-11-16T04:03:32,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21fb8fd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:32,543 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:32,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:32,544 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:32,572 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:32,573 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:32,590 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:32,604 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:32,604 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x8aa738cd, L:/127.0.0.1:35993 - R:/127.0.0.1:36664] 2024-11-16T04:03:32,639 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:32,640 WARN [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:36664; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:32,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:32,643 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:35993 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:03:32,643 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35993 2024-11-16T04:03:32,648 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=23 (was 23), OpenFileDescriptor=297 (was 297), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=136 (was 136), ProcessCount=11 (was 11), AvailableMemoryMB=7438 (was 7444) 2024-11-16T04:03:32,652 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=23, OpenFileDescriptor=297, MaxFileDescriptor=1048576, SystemLoadAverage=136, ProcessCount=11, AvailableMemoryMB=7438 2024-11-16T04:03:32,656 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:32,656 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35919 2024-11-16T04:03:32,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e7febc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:32,657 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:32,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:32,661 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:32,689 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:32,689 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:32,703 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:32,717 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:32,718 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xc2e9888b, L:/127.0.0.1:35919 - R:/127.0.0.1:36428] 2024-11-16T04:03:32,751 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:03:32,754 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:03:32,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:32,761 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=23 (was 23), OpenFileDescriptor=298 (was 297) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=136 (was 136), ProcessCount=11 (was 11), AvailableMemoryMB=7431 (was 7438) 2024-11-16T04:03:32,766 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=23, OpenFileDescriptor=298, MaxFileDescriptor=1048576, SystemLoadAverage=136, ProcessCount=11, AvailableMemoryMB=7431 2024-11-16T04:03:32,768 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:32,769 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45267 2024-11-16T04:03:32,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d37c09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:32,770 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:32,771 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:32,771 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:32,798 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:32,798 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:32,812 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:32,826 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:32,826 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x9abc389e, L:/127.0.0.1:45267 - R:/127.0.0.1:55874] 2024-11-16T04:03:32,860 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:03:32,861 WARN [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:55874 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:55874 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:32,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:32,868 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=23 (was 23), OpenFileDescriptor=299 (was 298) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=136 (was 136), ProcessCount=11 (was 11), AvailableMemoryMB=7425 (was 7431) 2024-11-16T04:03:32,873 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=23, OpenFileDescriptor=299, MaxFileDescriptor=1048576, SystemLoadAverage=136, ProcessCount=11, AvailableMemoryMB=7425 2024-11-16T04:03:32,876 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:32,877 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34467 2024-11-16T04:03:32,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62b7e3a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:32,878 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:32,880 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:32,880 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:32,909 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:32,910 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:32,924 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:32,937 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:32,937 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x7eb12e57, L:/127.0.0.1:34467 - R:/127.0.0.1:39452] 2024-11-16T04:03:32,973 WARN [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:39452 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:39452 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:32,975 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:32,975 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34467 2024-11-16T04:03:32,980 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=23 (was 23), OpenFileDescriptor=299 (was 299), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=136 (was 136), ProcessCount=11 (was 11), AvailableMemoryMB=7418 (was 7425) 2024-11-16T04:03:32,985 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=23, OpenFileDescriptor=299, MaxFileDescriptor=1048576, SystemLoadAverage=136, ProcessCount=11, AvailableMemoryMB=7418 2024-11-16T04:03:32,987 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:32,988 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44185 2024-11-16T04:03:32,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27836092, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:32,988 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:32,989 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:32,989 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:33,015 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:33,016 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:33,029 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:33,044 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:33,044 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x56e759a3, L:/127.0.0.1:44185 - R:/127.0.0.1:39562] 2024-11-16T04:03:33,076 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:33,179 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:33,181 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:33,182 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:33,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:33,190 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=24 (was 23) - Thread LEAK? -, OpenFileDescriptor=301 (was 299) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=136 (was 136), ProcessCount=11 (was 11), AvailableMemoryMB=7411 (was 7418) 2024-11-16T04:03:33,196 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=24, OpenFileDescriptor=300, MaxFileDescriptor=1048576, SystemLoadAverage=136, ProcessCount=11, AvailableMemoryMB=7411 2024-11-16T04:03:33,199 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:33,200 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36049 2024-11-16T04:03:33,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e4d74e0, compressor=org.apache.hadoop.io.compress.GzipCodec@2b0c2f3c, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:33,202 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:33,203 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:33,203 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:33,233 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:33,233 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:33,249 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:33,262 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:33,263 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x70f27a24, L:/127.0.0.1:36049 - R:/127.0.0.1:33754] 2024-11-16T04:03:33,302 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:33,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:33,317 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36049 2024-11-16T04:03:33,323 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=24 (was 24), OpenFileDescriptor=300 (was 300), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=136 (was 136), ProcessCount=11 (was 11), AvailableMemoryMB=7403 (was 7411) 2024-11-16T04:03:33,329 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=24, OpenFileDescriptor=300, MaxFileDescriptor=1048576, SystemLoadAverage=136, ProcessCount=11, AvailableMemoryMB=7403 2024-11-16T04:03:33,330 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:33,331 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36067 2024-11-16T04:03:33,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b2592d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:33,331 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:33,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:33,333 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:33,359 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:33,360 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:33,373 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:33,386 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:33,387 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xccdf10f5, L:/127.0.0.1:36067 - R:/127.0.0.1:40948] 2024-11-16T04:03:33,418 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:33,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:33,421 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36067 2024-11-16T04:03:33,426 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=24 (was 24), OpenFileDescriptor=300 (was 300), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=136 (was 136), ProcessCount=11 (was 11), AvailableMemoryMB=7396 (was 7403) 2024-11-16T04:03:33,432 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=24, OpenFileDescriptor=300, MaxFileDescriptor=1048576, SystemLoadAverage=136, ProcessCount=11, AvailableMemoryMB=7396 2024-11-16T04:03:33,434 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:33,434 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37601 2024-11-16T04:03:33,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57259077, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:33,435 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:33,436 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:33,436 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:33,467 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:33,467 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:33,482 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:33,495 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:33,495 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xdf9d043a, L:/127.0.0.1:37601 - R:/127.0.0.1:54378] 2024-11-16T04:03:33,527 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:33,529 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:33,529 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:54378 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:33,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:33,530 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:33,531 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37601 2024-11-16T04:03:33,536 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=24 (was 24), OpenFileDescriptor=300 (was 300), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=136 (was 136), ProcessCount=11 (was 11), AvailableMemoryMB=7390 (was 7396) 2024-11-16T04:03:33,541 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=24, OpenFileDescriptor=300, MaxFileDescriptor=1048576, SystemLoadAverage=136, ProcessCount=11, AvailableMemoryMB=7389 2024-11-16T04:03:33,543 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:33,544 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40529 2024-11-16T04:03:33,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:33,544 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:33,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:33,545 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:33,574 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:33,575 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:33,588 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:33,601 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:33,601 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x6e33417e, L:/127.0.0.1:40529 - R:/127.0.0.1:50248] 2024-11-16T04:03:33,632 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:33,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:33,634 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40529 2024-11-16T04:03:33,639 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=24 (was 24), OpenFileDescriptor=300 (was 300), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=136 (was 136), ProcessCount=11 (was 11), AvailableMemoryMB=7383 (was 7389) 2024-11-16T04:03:33,644 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=24, OpenFileDescriptor=300, MaxFileDescriptor=1048576, SystemLoadAverage=136, ProcessCount=11, AvailableMemoryMB=7383 2024-11-16T04:03:33,647 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:33,647 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46135 2024-11-16T04:03:33,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bb155f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:33,648 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:33,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:33,653 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:33,685 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:33,685 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:33,698 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:33,711 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:33,711 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xba2c078c, L:/127.0.0.1:46135 - R:/127.0.0.1:35488] 2024-11-16T04:03:33,744 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:33,747 WARN [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:33,753 INFO [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-a276f83ae2a83705616ad9c15b9f2b38-f10ca13632dc9520-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:03:33,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:33,755 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:33,755 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46135 2024-11-16T04:03:33,760 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=24 (was 24), OpenFileDescriptor=300 (was 300), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=136 (was 136), ProcessCount=11 (was 11), AvailableMemoryMB=7376 (was 7383) 2024-11-16T04:03:33,764 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=24, OpenFileDescriptor=300, MaxFileDescriptor=1048576, SystemLoadAverage=136, ProcessCount=11, AvailableMemoryMB=7375 2024-11-16T04:03:33,766 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:33,767 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42683 2024-11-16T04:03:33,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71304fbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:33,767 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:33,768 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:33,769 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:33,811 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:33,811 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:33,830 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:33,844 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:33,844 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x0603b6bc, L:/127.0.0.1:42683 - R:/127.0.0.1:47374] 2024-11-16T04:03:33,883 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:33,918 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:42683 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=148ms, rpcTimeout=100ms 2024-11-16T04:03:33,918 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:42683 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=148ms, rpcTimeout=100ms 2024-11-16T04:03:33,918 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:42683 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=147ms, rpcTimeout=100ms 2024-11-16T04:03:33,918 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:42683 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=147ms, rpcTimeout=100ms 2024-11-16T04:03:33,918 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:42683 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=148ms, rpcTimeout=100ms 2024-11-16T04:03:33,918 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:42683 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=148ms, rpcTimeout=100ms 2024-11-16T04:03:33,919 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:42683 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=148ms, rpcTimeout=100ms 2024-11-16T04:03:33,919 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:42683 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=148ms, rpcTimeout=100ms 2024-11-16T04:03:33,919 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:42683 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=147ms, rpcTimeout=100ms 2024-11-16T04:03:33,919 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:42683 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=147ms, rpcTimeout=100ms 2024-11-16T04:03:33,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:33,919 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42683 2024-11-16T04:03:33,926 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[0: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=false, clientTlsEnabled=true] Thread=25 (was 24) - Thread LEAK? -, OpenFileDescriptor=300 (was 300), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=136 (was 136), ProcessCount=11 (was 11), AvailableMemoryMB=7368 (was 7375) 2024-11-16T04:03:33,932 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=25, OpenFileDescriptor=300, MaxFileDescriptor=1048576, SystemLoadAverage=136, ProcessCount=11, AvailableMemoryMB=7368 2024-11-16T04:03:33,934 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:33,935 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42147 2024-11-16T04:03:33,935 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@327e6442, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:33,935 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:33,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:33,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:33,937 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:42147 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:03:33,937 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42147 2024-11-16T04:03:33,943 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=25 (was 25), OpenFileDescriptor=300 (was 300), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=136 (was 136), ProcessCount=11 (was 11), AvailableMemoryMB=7367 (was 7368) 2024-11-16T04:03:33,948 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=25, OpenFileDescriptor=300, MaxFileDescriptor=1048576, SystemLoadAverage=136, ProcessCount=11, AvailableMemoryMB=7367 2024-11-16T04:03:33,949 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:33,950 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46759 2024-11-16T04:03:33,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675411c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:33,951 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:33,951 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:33,952 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:33,981 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:33,981 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:33,994 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:34,007 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:34,008 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x684af1b2, L:/127.0.0.1:46759 - R:/127.0.0.1:60082] 2024-11-16T04:03:34,041 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:34,086 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:46759 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:03:34,197 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:46759 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:34,307 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:46759 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:34,417 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:46759 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:03:34,527 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:46759 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:03:34,637 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:46759 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:34,747 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:46759 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:03:34,857 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:46759 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:03:34,884 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:47374 deadline: 1731729813984 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:47374 2024-11-16T04:03:34,884 WARN [TestNettyTlsIPC-1-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:34,885 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:47374 deadline: 1731729813984 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:47374 2024-11-16T04:03:34,885 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:47374 deadline: 1731729813985 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:47374 2024-11-16T04:03:34,885 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:47374 deadline: 1731729813985 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:47374 2024-11-16T04:03:34,885 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:47374 deadline: 1731729813986 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:47374 2024-11-16T04:03:34,886 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:47374 deadline: 1731729813986 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:47374 2024-11-16T04:03:34,886 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:47374 deadline: 1731729813986 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:47374 2024-11-16T04:03:34,886 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:47374 deadline: 1731729813987 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:47374 2024-11-16T04:03:34,887 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:47374 deadline: 1731729813987 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:47374 2024-11-16T04:03:34,967 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:46759 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:03:35,043 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:60082 deadline: 1731729814188 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:60082 2024-11-16T04:03:35,044 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:60082 deadline: 1731729814300 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:60082 2024-11-16T04:03:35,045 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:60082 deadline: 1731729814412 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:60082 2024-11-16T04:03:35,045 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:60082 deadline: 1731729814522 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:60082 2024-11-16T04:03:35,046 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:60082 deadline: 1731729814631 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:60082 2024-11-16T04:03:35,047 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:60082 deadline: 1731729814741 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:60082 2024-11-16T04:03:35,048 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:60082 deadline: 1731729814851 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:60082 2024-11-16T04:03:35,048 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:03:35,048 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:60082 deadline: 1731729814961 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:60082 2024-11-16T04:03:35,076 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:46759 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:35,077 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:35,077 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46759 2024-11-16T04:03:35,088 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=25 (was 25), OpenFileDescriptor=300 (was 300), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 136), ProcessCount=11 (was 11), AvailableMemoryMB=7356 (was 7367) 2024-11-16T04:03:35,096 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=25, OpenFileDescriptor=300, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7356 2024-11-16T04:03:35,097 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:35,098 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45837 2024-11-16T04:03:35,098 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@434b270c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:35,098 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:35,099 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:35,099 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:35,128 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:35,129 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:35,143 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:35,156 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:35,157 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xa5a8af9b, L:/127.0.0.1:45837 - R:/127.0.0.1:51606] 2024-11-16T04:03:35,187 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:35,188 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:35,188 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:51606 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:35,189 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:35,190 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:35,191 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:35,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:35,197 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26 (was 25) - Thread LEAK? -, OpenFileDescriptor=301 (was 300) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7349 (was 7356) 2024-11-16T04:03:35,201 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26, OpenFileDescriptor=301, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7349 2024-11-16T04:03:35,202 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:35,203 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33271 2024-11-16T04:03:35,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d50e14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:35,203 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:35,204 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:35,205 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:35,230 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:35,230 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:35,244 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:35,257 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:35,258 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x944663f0, L:/127.0.0.1:33271 - R:/127.0.0.1:40246] 2024-11-16T04:03:35,288 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:35,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:35,307 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33271 2024-11-16T04:03:35,312 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26 (was 26), OpenFileDescriptor=301 (was 301), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7343 (was 7349) 2024-11-16T04:03:35,316 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26, OpenFileDescriptor=301, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7343 2024-11-16T04:03:35,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aef094e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:35,318 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:35,318 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33507 2024-11-16T04:03:35,318 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:35,319 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:35,320 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:35,347 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:35,348 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:35,362 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:35,376 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:35,376 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x1b7355ba, L:/127.0.0.1:33507 - R:/127.0.0.1:59698] 2024-11-16T04:03:35,405 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:35,406 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:35,407 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:59698 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:35,407 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:35,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:35,408 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33507 2024-11-16T04:03:35,412 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26 (was 26), OpenFileDescriptor=301 (was 301), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7336 (was 7343) 2024-11-16T04:03:35,417 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26, OpenFileDescriptor=301, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7336 2024-11-16T04:03:35,419 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:35,419 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46563 2024-11-16T04:03:35,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45978a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:35,420 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:35,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:35,421 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:35,449 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:35,450 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:35,463 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:35,477 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:35,477 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x9700c52e, L:/127.0.0.1:46563 - R:/127.0.0.1:57358] 2024-11-16T04:03:35,510 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:35,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:35,516 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46563 2024-11-16T04:03:35,521 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26 (was 26), OpenFileDescriptor=301 (was 301), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7328 (was 7336) 2024-11-16T04:03:35,526 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26, OpenFileDescriptor=301, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7328 2024-11-16T04:03:35,527 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:35,528 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34941 2024-11-16T04:03:35,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27c6d0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:35,528 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:35,529 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:35,529 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:35,559 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:35,559 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:35,573 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:35,586 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:35,587 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x0b47fab4, L:/127.0.0.1:34941 - R:/127.0.0.1:53104] 2024-11-16T04:03:35,617 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:35,617 WARN [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:53104; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:35,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:35,618 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:34941 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:03:35,619 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34941 2024-11-16T04:03:35,624 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26 (was 26), OpenFileDescriptor=301 (was 301), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7322 (was 7328) 2024-11-16T04:03:35,628 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26, OpenFileDescriptor=301, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7322 2024-11-16T04:03:35,630 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:35,630 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45863 2024-11-16T04:03:35,631 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45828444, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:35,631 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:35,631 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:35,632 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:35,659 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:35,660 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:35,685 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:35,698 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:35,698 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x27c45b6f, L:/127.0.0.1:45863 - R:/127.0.0.1:51204] 2024-11-16T04:03:35,730 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:03:35,731 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:03:35,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:35,736 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26 (was 26), OpenFileDescriptor=302 (was 301) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7316 (was 7322) 2024-11-16T04:03:35,741 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26, OpenFileDescriptor=302, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7315 2024-11-16T04:03:35,742 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:35,743 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45827 2024-11-16T04:03:35,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6242ca34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:35,743 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:35,744 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:35,744 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:35,771 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:35,772 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:35,785 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:35,798 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:35,798 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x0703d73c, L:/127.0.0.1:45827 - R:/127.0.0.1:39442] 2024-11-16T04:03:35,828 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:03:35,828 WARN [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:39442 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:39442 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:35,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:35,834 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26 (was 26), OpenFileDescriptor=303 (was 302) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7310 (was 7315) 2024-11-16T04:03:35,839 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26, OpenFileDescriptor=303, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7309 2024-11-16T04:03:35,840 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:35,841 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40431 2024-11-16T04:03:35,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@126e9f17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:35,841 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:35,842 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:35,842 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:35,871 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:35,871 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:35,884 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:35,898 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:35,898 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xa8dc5bff, L:/127.0.0.1:40431 - R:/127.0.0.1:35180] 2024-11-16T04:03:35,928 WARN [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:35180 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:35180 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:35,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:35,930 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40431 2024-11-16T04:03:35,934 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26 (was 26), OpenFileDescriptor=303 (was 303), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7303 (was 7309) 2024-11-16T04:03:35,939 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26, OpenFileDescriptor=303, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7303 2024-11-16T04:03:35,940 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:35,941 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34573 2024-11-16T04:03:35,941 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@435ec3b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:35,941 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:35,942 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:35,942 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:35,968 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:35,969 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:35,983 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:35,996 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:35,996 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xe33fb761, L:/127.0.0.1:34573 - R:/127.0.0.1:37168] 2024-11-16T04:03:36,025 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:36,050 WARN [TestNettyTlsIPC-1-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:36,128 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:36,129 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:36,130 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:36,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:36,136 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26 (was 26), OpenFileDescriptor=304 (was 303) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7297 (was 7303) 2024-11-16T04:03:36,142 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26, OpenFileDescriptor=304, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7296 2024-11-16T04:03:36,144 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:36,144 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43323 2024-11-16T04:03:36,145 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ac46350, compressor=org.apache.hadoop.io.compress.GzipCodec@bc85891, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:36,145 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:36,146 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:36,146 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:36,177 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:36,177 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:36,191 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:36,204 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:36,205 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x2b7a40da, L:/127.0.0.1:43323 - R:/127.0.0.1:56030] 2024-11-16T04:03:36,234 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:36,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:36,236 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43323 2024-11-16T04:03:36,241 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26 (was 26), OpenFileDescriptor=304 (was 304), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7290 (was 7296) 2024-11-16T04:03:36,245 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26, OpenFileDescriptor=304, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7290 2024-11-16T04:03:36,247 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:36,247 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35031 2024-11-16T04:03:36,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cdb6359, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:36,247 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:36,248 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:36,248 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:36,277 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:36,278 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:36,292 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:36,304 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:36,304 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xcf444327, L:/127.0.0.1:35031 - R:/127.0.0.1:47434] 2024-11-16T04:03:36,332 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:36,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:36,334 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35031 2024-11-16T04:03:36,340 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26 (was 26), OpenFileDescriptor=304 (was 304), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7284 (was 7290) 2024-11-16T04:03:36,344 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26, OpenFileDescriptor=304, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7284 2024-11-16T04:03:36,345 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:36,345 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41141 2024-11-16T04:03:36,346 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3326d6b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:36,346 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:36,346 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:36,347 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:36,375 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:36,376 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:36,391 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:36,403 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:36,404 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x1f2e25c0, L:/127.0.0.1:41141 - R:/127.0.0.1:54340] 2024-11-16T04:03:36,444 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:36,446 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:36,446 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:54340 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:36,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:36,447 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:36,447 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41141 2024-11-16T04:03:36,452 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26 (was 26), OpenFileDescriptor=304 (was 304), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7278 (was 7284) 2024-11-16T04:03:36,457 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26, OpenFileDescriptor=304, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7277 2024-11-16T04:03:36,459 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:36,459 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35829 2024-11-16T04:03:36,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:36,459 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:36,460 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:36,461 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:36,491 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:36,492 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:36,507 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:36,520 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:36,520 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x9f064029, L:/127.0.0.1:35829 - R:/127.0.0.1:39906] 2024-11-16T04:03:36,552 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:36,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:36,554 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35829 2024-11-16T04:03:36,559 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26 (was 26), OpenFileDescriptor=304 (was 304), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7271 (was 7277) 2024-11-16T04:03:36,563 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26, OpenFileDescriptor=304, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7270 2024-11-16T04:03:36,565 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:36,566 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41275 2024-11-16T04:03:36,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61578a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:36,566 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:36,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:36,567 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:36,597 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:36,597 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:36,612 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:36,625 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:36,625 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x7b515df7, L:/127.0.0.1:41275 - R:/127.0.0.1:60578] 2024-11-16T04:03:36,654 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:36,654 WARN [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:36,654 INFO [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-63a657e78e955feee2c3350c724b237f-46a7d0954bf7f4be-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:03:36,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:36,655 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:36,656 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41275 2024-11-16T04:03:36,660 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26 (was 26), OpenFileDescriptor=304 (was 304), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7264 (was 7270) 2024-11-16T04:03:36,665 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=26, OpenFileDescriptor=304, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7264 2024-11-16T04:03:36,667 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:36,667 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38733 2024-11-16T04:03:36,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@375b99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:36,668 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:36,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:36,669 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:36,700 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:36,700 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:36,714 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:36,727 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:36,727 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x2ce08d36, L:/127.0.0.1:38733 - R:/127.0.0.1:49170] 2024-11-16T04:03:36,760 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:36,807 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:38733 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=137ms, rpcTimeout=100ms 2024-11-16T04:03:36,807 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:38733 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=137ms, rpcTimeout=100ms 2024-11-16T04:03:36,807 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:38733 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=137ms, rpcTimeout=100ms 2024-11-16T04:03:36,807 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:38733 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=137ms, rpcTimeout=100ms 2024-11-16T04:03:36,807 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:38733 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=137ms, rpcTimeout=100ms 2024-11-16T04:03:36,807 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:38733 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=137ms, rpcTimeout=100ms 2024-11-16T04:03:36,808 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:38733 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=137ms, rpcTimeout=100ms 2024-11-16T04:03:36,808 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:38733 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=136ms, rpcTimeout=100ms 2024-11-16T04:03:36,808 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:38733 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=137ms, rpcTimeout=100ms 2024-11-16T04:03:36,808 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:38733 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=137ms, rpcTimeout=100ms 2024-11-16T04:03:36,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:36,808 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38733 2024-11-16T04:03:36,816 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[1: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=true] Thread=27 (was 26) - Thread LEAK? -, OpenFileDescriptor=304 (was 304), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7257 (was 7264) 2024-11-16T04:03:36,821 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=27, OpenFileDescriptor=304, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7257 2024-11-16T04:03:36,823 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:36,823 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36297 2024-11-16T04:03:36,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f06fa23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:36,824 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:36,825 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:36,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:36,825 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:36297 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:03:36,826 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36297 2024-11-16T04:03:36,831 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=27 (was 27), OpenFileDescriptor=304 (was 304), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7255 (was 7257) 2024-11-16T04:03:36,836 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=27, OpenFileDescriptor=304, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7255 2024-11-16T04:03:36,837 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:36,838 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33101 2024-11-16T04:03:36,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c40dabf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:36,838 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:36,839 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:36,857 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:36,870 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:36,870 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x79c773ba, L:/127.0.0.1:33101 - R:/127.0.0.1:53540] 2024-11-16T04:03:36,871 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:36,947 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33101 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=107ms, rpcTimeout=100ms 2024-11-16T04:03:37,056 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33101 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:37,167 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33101 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:37,276 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33101 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:37,386 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33101 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:37,496 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33101 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:37,607 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33101 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:37,717 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33101 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:37,762 WARN [TestNettyTlsIPC-1-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:37,762 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49170 deadline: 1731729816861 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49170 2024-11-16T04:03:37,763 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49170 deadline: 1731729816862 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49170 2024-11-16T04:03:37,764 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49170 deadline: 1731729816862 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49170 2024-11-16T04:03:37,765 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49170 deadline: 1731729816862 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49170 2024-11-16T04:03:37,765 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49170 deadline: 1731729816863 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49170 2024-11-16T04:03:37,766 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49170 deadline: 1731729816863 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49170 2024-11-16T04:03:37,767 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49170 deadline: 1731729816863 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49170 2024-11-16T04:03:37,767 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49170 deadline: 1731729816864 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49170 2024-11-16T04:03:37,767 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49170 deadline: 1731729816864 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49170 2024-11-16T04:03:37,826 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33101 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:37,873 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53540 deadline: 1731729817048 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53540 2024-11-16T04:03:37,873 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:03:37,873 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53540 deadline: 1731729817157 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53540 2024-11-16T04:03:37,874 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53540 deadline: 1731729817268 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53540 2024-11-16T04:03:37,874 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53540 deadline: 1731729817379 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53540 2024-11-16T04:03:37,875 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53540 deadline: 1731729817489 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53540 2024-11-16T04:03:37,875 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53540 deadline: 1731729817599 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53540 2024-11-16T04:03:37,875 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53540 deadline: 1731729817709 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53540 2024-11-16T04:03:37,876 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53540 deadline: 1731729817819 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53540 2024-11-16T04:03:37,936 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33101 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:37,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:37,936 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33101 2024-11-16T04:03:37,950 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=27 (was 27), OpenFileDescriptor=304 (was 304), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7252 (was 7255) 2024-11-16T04:03:37,956 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=27, OpenFileDescriptor=304, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7251 2024-11-16T04:03:37,957 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:37,958 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43033 2024-11-16T04:03:37,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2216c5e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:37,958 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:37,959 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:37,975 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:37,988 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:37,988 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x1c621cea, L:/127.0.0.1:43033 - R:/127.0.0.1:54476] 2024-11-16T04:03:37,989 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:37,990 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:37,990 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:54476 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:37,991 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:37,992 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:37,992 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:37,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:37,997 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=28 (was 27) - Thread LEAK? -, OpenFileDescriptor=305 (was 304) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7248 (was 7251) 2024-11-16T04:03:38,002 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=28, OpenFileDescriptor=305, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7248 2024-11-16T04:03:38,003 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:38,004 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46125 2024-11-16T04:03:38,004 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ea5e4b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:38,004 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:38,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:38,019 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:38,032 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:38,033 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x6611ae3a, L:/127.0.0.1:46125 - R:/127.0.0.1:33988] 2024-11-16T04:03:38,033 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:38,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:38,046 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46125 2024-11-16T04:03:38,052 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=28 (was 28), OpenFileDescriptor=305 (was 305), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7244 (was 7248) 2024-11-16T04:03:38,056 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=28, OpenFileDescriptor=305, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7243 2024-11-16T04:03:38,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6200ef2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:38,058 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:38,058 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41429 2024-11-16T04:03:38,058 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:38,059 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:38,075 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:38,089 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:38,089 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xc8953189, L:/127.0.0.1:41429 - R:/127.0.0.1:38542] 2024-11-16T04:03:38,089 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:38,090 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:38,091 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:38542 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:38,091 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:38,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:38,091 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41429 2024-11-16T04:03:38,096 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=28 (was 28), OpenFileDescriptor=305 (was 305), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7240 (was 7243) 2024-11-16T04:03:38,100 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=28, OpenFileDescriptor=305, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7239 2024-11-16T04:03:38,102 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:38,102 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43891 2024-11-16T04:03:38,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ff81998, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:38,102 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:38,103 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:38,120 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:38,134 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:38,134 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xffdc6c4e, L:/127.0.0.1:43891 - R:/127.0.0.1:60378] 2024-11-16T04:03:38,135 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:38,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:38,140 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43891 2024-11-16T04:03:38,145 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=28 (was 28), OpenFileDescriptor=305 (was 305), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7236 (was 7239) 2024-11-16T04:03:38,149 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=28, OpenFileDescriptor=305, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7235 2024-11-16T04:03:38,150 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:38,150 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45065 2024-11-16T04:03:38,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b5cc71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:38,150 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:38,151 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:38,168 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:38,181 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:38,181 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xf2fd5415, L:/127.0.0.1:45065 - R:/127.0.0.1:40494] 2024-11-16T04:03:38,181 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:38,181 WARN [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:40494; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:38,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:38,182 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:45065 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:03:38,182 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45065 2024-11-16T04:03:38,187 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=28 (was 28), OpenFileDescriptor=305 (was 305), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7233 (was 7235) 2024-11-16T04:03:38,192 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=28, OpenFileDescriptor=305, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7232 2024-11-16T04:03:38,194 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:38,194 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36973 2024-11-16T04:03:38,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dba76c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:38,194 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:38,195 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:38,213 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:38,226 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:38,226 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x923e02ac, L:/127.0.0.1:36973 - R:/127.0.0.1:43796] 2024-11-16T04:03:38,226 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:03:38,227 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:03:38,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:38,231 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=28 (was 28), OpenFileDescriptor=306 (was 305) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7230 (was 7232) 2024-11-16T04:03:38,236 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=28, OpenFileDescriptor=306, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7229 2024-11-16T04:03:38,237 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:38,237 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40499 2024-11-16T04:03:38,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57822f1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:38,237 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:38,238 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:38,254 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:38,266 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:38,267 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x91c35072, L:/127.0.0.1:40499 - R:/127.0.0.1:43566] 2024-11-16T04:03:38,267 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:03:38,267 WARN [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:43566 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:43566 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:38,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:38,272 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=28 (was 28), OpenFileDescriptor=307 (was 306) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7226 (was 7229) 2024-11-16T04:03:38,277 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=28, OpenFileDescriptor=307, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7226 2024-11-16T04:03:38,278 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:38,278 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33725 2024-11-16T04:03:38,278 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60d29bba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:38,279 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:38,279 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:38,296 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:38,310 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:38,311 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xc19dc08d, L:/127.0.0.1:33725 - R:/127.0.0.1:39368] 2024-11-16T04:03:38,311 WARN [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:39368 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:39368 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:38,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:38,312 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33725 2024-11-16T04:03:38,318 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=28 (was 28), OpenFileDescriptor=307 (was 307), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7223 (was 7226) 2024-11-16T04:03:38,322 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=28, OpenFileDescriptor=307, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7223 2024-11-16T04:03:38,324 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:38,324 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41053 2024-11-16T04:03:38,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13bf4136, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:38,324 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:38,325 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:38,342 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:38,355 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:38,355 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xef6f71b3, L:/127.0.0.1:41053 - R:/127.0.0.1:38966] 2024-11-16T04:03:38,355 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:38,458 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:38,460 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:38,461 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:38,461 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:38,468 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=29 (was 28) - Thread LEAK? -, OpenFileDescriptor=308 (was 307) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7220 (was 7223) 2024-11-16T04:03:38,473 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=29, OpenFileDescriptor=308, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7220 2024-11-16T04:03:38,475 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:38,475 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37963 2024-11-16T04:03:38,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d49f3d, compressor=org.apache.hadoop.io.compress.GzipCodec@12c33d4d, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:38,476 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:38,476 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:38,498 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:38,510 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:38,511 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xe018dc00, L:/127.0.0.1:37963 - R:/127.0.0.1:38378] 2024-11-16T04:03:38,511 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:38,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:38,513 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37963 2024-11-16T04:03:38,518 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=29 (was 29), OpenFileDescriptor=308 (was 308), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7218 (was 7220) 2024-11-16T04:03:38,523 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=29, OpenFileDescriptor=308, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7217 2024-11-16T04:03:38,524 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:38,525 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43609 2024-11-16T04:03:38,525 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd9158a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:38,525 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:38,525 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:38,544 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:38,556 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:38,557 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x05bb010e, L:/127.0.0.1:43609 - R:/127.0.0.1:53130] 2024-11-16T04:03:38,557 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:38,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:38,559 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43609 2024-11-16T04:03:38,564 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=29 (was 29), OpenFileDescriptor=308 (was 308), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7214 (was 7217) 2024-11-16T04:03:38,568 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=29, OpenFileDescriptor=308, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7213 2024-11-16T04:03:38,570 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:38,570 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38013 2024-11-16T04:03:38,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@739469d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:38,571 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:38,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:38,585 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:38,598 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:38,598 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xcb0b3778, L:/127.0.0.1:38013 - R:/127.0.0.1:58504] 2024-11-16T04:03:38,599 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:38,600 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:38,600 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:58504 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:38,601 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:38,601 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:38,601 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38013 2024-11-16T04:03:38,606 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=29 (was 29), OpenFileDescriptor=308 (was 308), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7211 (was 7213) 2024-11-16T04:03:38,610 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=29, OpenFileDescriptor=308, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7211 2024-11-16T04:03:38,611 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:38,611 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34811 2024-11-16T04:03:38,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:38,612 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:38,612 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:38,629 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:38,642 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:38,642 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x265802d4, L:/127.0.0.1:34811 - R:/127.0.0.1:53538] 2024-11-16T04:03:38,643 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:38,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:38,644 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34811 2024-11-16T04:03:38,649 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=29 (was 29), OpenFileDescriptor=308 (was 308), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7209 (was 7211) 2024-11-16T04:03:38,654 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=29, OpenFileDescriptor=308, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7209 2024-11-16T04:03:38,655 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:38,655 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44935 2024-11-16T04:03:38,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13effefc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:38,656 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:38,656 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:38,674 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:38,687 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:38,687 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xc0f1e0eb, L:/127.0.0.1:44935 - R:/127.0.0.1:47460] 2024-11-16T04:03:38,687 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:38,687 WARN [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:38,688 INFO [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-0c5b80ab30797bb26592a675ca578957-d833bd0a7d9302e6-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:03:38,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:38,688 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:38,689 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44935 2024-11-16T04:03:38,693 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=29 (was 29), OpenFileDescriptor=308 (was 308), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7208 (was 7209) 2024-11-16T04:03:38,698 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=29, OpenFileDescriptor=308, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7206 2024-11-16T04:03:38,700 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:38,700 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38913 2024-11-16T04:03:38,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@327d0bb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:38,700 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:38,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:38,714 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:38,727 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:38,727 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xf20f73cf, L:/127.0.0.1:38913 - R:/127.0.0.1:52056] 2024-11-16T04:03:38,728 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:52056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:38,809 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:38913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=105ms, rpcTimeout=100ms 2024-11-16T04:03:38,809 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:38913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=105ms, rpcTimeout=100ms 2024-11-16T04:03:38,810 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:38913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=105ms, rpcTimeout=100ms 2024-11-16T04:03:38,810 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:38913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=105ms, rpcTimeout=100ms 2024-11-16T04:03:38,810 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:38913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=105ms, rpcTimeout=100ms 2024-11-16T04:03:38,810 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:38913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=105ms, rpcTimeout=100ms 2024-11-16T04:03:38,810 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:38913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=106ms, rpcTimeout=100ms 2024-11-16T04:03:38,810 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:38913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=106ms, rpcTimeout=100ms 2024-11-16T04:03:38,811 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:38913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=106ms, rpcTimeout=100ms 2024-11-16T04:03:38,811 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:38913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=107ms, rpcTimeout=100ms 2024-11-16T04:03:38,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:38,811 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38913 2024-11-16T04:03:38,820 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[2: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@741b3bc3, acceptPlainText=true, clientTlsEnabled=false] Thread=30 (was 29) - Thread LEAK? -, OpenFileDescriptor=308 (was 308), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7203 (was 7206) 2024-11-16T04:03:38,825 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30, OpenFileDescriptor=308, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7203 2024-11-16T04:03:38,877 WARN [TestNettyTlsIPC-1-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:39,070 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:39,071 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40675 2024-11-16T04:03:39,071 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51dfea42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:39,071 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:39,072 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:39,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:39,072 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:40675 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:03:39,072 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40675 2024-11-16T04:03:39,077 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=29 (was 30), OpenFileDescriptor=308 (was 308), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7177 (was 7203) 2024-11-16T04:03:39,082 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=29, OpenFileDescriptor=308, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7177 2024-11-16T04:03:39,083 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:39,084 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45865 2024-11-16T04:03:39,084 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ab49ebf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:39,084 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:39,084 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:39,085 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:39,117 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:39,118 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:39,132 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:39,145 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:39,146 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x0f53bc50, L:/127.0.0.1:45865 - R:/127.0.0.1:53992] 2024-11-16T04:03:39,187 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:39,226 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45865 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=142ms, rpcTimeout=100ms 2024-11-16T04:03:39,336 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45865 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:39,446 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45865 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:39,557 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45865 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:39,667 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45865 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:39,729 WARN [TestNettyTlsIPC-1-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:39,729 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52056 deadline: 1731729818829 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52056 2024-11-16T04:03:39,730 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52056 deadline: 1731729818829 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52056 2024-11-16T04:03:39,730 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52056 deadline: 1731729818829 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52056 2024-11-16T04:03:39,730 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52056 deadline: 1731729818829 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52056 2024-11-16T04:03:39,730 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52056 deadline: 1731729818829 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52056 2024-11-16T04:03:39,731 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52056 deadline: 1731729818830 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52056 2024-11-16T04:03:39,731 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52056 deadline: 1731729818830 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52056 2024-11-16T04:03:39,731 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52056 deadline: 1731729818830 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52056 2024-11-16T04:03:39,731 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52056 deadline: 1731729818830 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52056 2024-11-16T04:03:39,776 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45865 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:39,887 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45865 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:39,997 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45865 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:40,107 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45865 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:40,189 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53992 deadline: 1731729819327 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53992 2024-11-16T04:03:40,190 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:03:40,190 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53992 deadline: 1731729819438 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53992 2024-11-16T04:03:40,190 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53992 deadline: 1731729819549 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53992 2024-11-16T04:03:40,191 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53992 deadline: 1731729819660 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53992 2024-11-16T04:03:40,192 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53992 deadline: 1731729819770 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53992 2024-11-16T04:03:40,192 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53992 deadline: 1731729819878 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53992 2024-11-16T04:03:40,193 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53992 deadline: 1731729819988 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53992 2024-11-16T04:03:40,193 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53992 deadline: 1731729820099 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53992 2024-11-16T04:03:40,216 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45865 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:40,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:40,217 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45865 2024-11-16T04:03:40,225 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=29 (was 29), OpenFileDescriptor=308 (was 308), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 125) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7169 (was 7177) 2024-11-16T04:03:40,234 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=29, OpenFileDescriptor=308, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7169 2024-11-16T04:03:40,235 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:40,235 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43205 2024-11-16T04:03:40,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd3301e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:40,235 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:40,236 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:40,236 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:40,261 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:40,262 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:40,275 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:40,288 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:40,288 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xa2f937bd, L:/127.0.0.1:43205 - R:/127.0.0.1:40290] 2024-11-16T04:03:40,317 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:40,318 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:40,319 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:40290 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:40,320 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:40,321 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:40,321 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:40,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:40,326 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30 (was 29) - Thread LEAK? -, OpenFileDescriptor=309 (was 308) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7163 (was 7169) 2024-11-16T04:03:40,331 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30, OpenFileDescriptor=309, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7163 2024-11-16T04:03:40,333 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:40,333 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33801 2024-11-16T04:03:40,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dcb79e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:40,334 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:40,335 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:40,335 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:40,365 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:40,365 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:40,379 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:40,392 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:40,392 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xc54afd60, L:/127.0.0.1:33801 - R:/127.0.0.1:38714] 2024-11-16T04:03:40,421 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:40,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:40,436 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33801 2024-11-16T04:03:40,441 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30 (was 30), OpenFileDescriptor=309 (was 309), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7156 (was 7163) 2024-11-16T04:03:40,446 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30, OpenFileDescriptor=309, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7155 2024-11-16T04:03:40,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c6db1f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:40,447 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:40,447 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45557 2024-11-16T04:03:40,447 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:40,448 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:40,448 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:40,474 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:40,474 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:40,487 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:40,499 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:40,500 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x37067238, L:/127.0.0.1:45557 - R:/127.0.0.1:44216] 2024-11-16T04:03:40,528 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:40,530 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:40,530 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:44216 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:40,531 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:40,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:40,531 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45557 2024-11-16T04:03:40,536 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30 (was 30), OpenFileDescriptor=309 (was 309), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7149 (was 7155) 2024-11-16T04:03:40,540 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30, OpenFileDescriptor=309, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7149 2024-11-16T04:03:40,541 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:40,541 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37941 2024-11-16T04:03:40,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f234da5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:40,542 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:40,542 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:40,543 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:40,568 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:40,569 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:40,582 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:40,595 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:40,595 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xe470f02f, L:/127.0.0.1:37941 - R:/127.0.0.1:43088] 2024-11-16T04:03:40,624 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:40,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:40,629 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37941 2024-11-16T04:03:40,634 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30 (was 30), OpenFileDescriptor=309 (was 309), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7142 (was 7149) 2024-11-16T04:03:40,638 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30, OpenFileDescriptor=309, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7141 2024-11-16T04:03:40,639 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:40,640 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42517 2024-11-16T04:03:40,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a37e73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:40,640 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:40,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:40,641 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:40,666 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:40,667 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:40,680 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:40,693 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:40,693 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x9ec1abda, L:/127.0.0.1:42517 - R:/127.0.0.1:50240] 2024-11-16T04:03:40,721 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:40,721 WARN [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:50240; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:40,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:40,723 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:42517 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:03:40,723 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42517 2024-11-16T04:03:40,727 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30 (was 30), OpenFileDescriptor=309 (was 309), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7136 (was 7141) 2024-11-16T04:03:40,731 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30, OpenFileDescriptor=309, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7136 2024-11-16T04:03:40,732 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:40,733 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42845 2024-11-16T04:03:40,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@402acab2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:40,733 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:40,733 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:40,734 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:40,761 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:40,761 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:40,775 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:40,789 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:40,789 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x32561d02, L:/127.0.0.1:42845 - R:/127.0.0.1:43062] 2024-11-16T04:03:40,817 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:03:40,817 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:03:40,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:40,823 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30 (was 30), OpenFileDescriptor=310 (was 309) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7130 (was 7136) 2024-11-16T04:03:40,827 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30, OpenFileDescriptor=310, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7130 2024-11-16T04:03:40,828 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:40,829 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38521 2024-11-16T04:03:40,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@710b73ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:40,829 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:40,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:40,830 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:40,858 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:40,858 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:40,872 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:40,885 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:40,885 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xb8ee4375, L:/127.0.0.1:38521 - R:/127.0.0.1:58530] 2024-11-16T04:03:40,912 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:03:40,912 WARN [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:58530 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:58530 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:40,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:40,918 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30 (was 30), OpenFileDescriptor=311 (was 310) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7123 (was 7130) 2024-11-16T04:03:40,922 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30, OpenFileDescriptor=311, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7123 2024-11-16T04:03:40,924 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:40,924 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41515 2024-11-16T04:03:40,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c2bb2e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:40,924 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:40,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:40,925 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:40,951 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:40,951 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:40,964 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:40,977 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:40,977 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xd5a306e7, L:/127.0.0.1:41515 - R:/127.0.0.1:41026] 2024-11-16T04:03:41,005 WARN [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:41026 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:41026 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:41,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:41,007 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41515 2024-11-16T04:03:41,011 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30 (was 30), OpenFileDescriptor=311 (was 311), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7117 (was 7123) 2024-11-16T04:03:41,016 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30, OpenFileDescriptor=311, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7116 2024-11-16T04:03:41,017 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:41,018 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39915 2024-11-16T04:03:41,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1f5118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:41,018 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:41,019 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:41,020 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:41,050 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:41,050 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:41,064 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:41,077 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:41,077 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x315f2784, L:/127.0.0.1:39915 - R:/127.0.0.1:58098] 2024-11-16T04:03:41,104 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:41,195 WARN [TestNettyTlsIPC-1-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:41,207 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:41,210 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:41,211 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:41,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:41,217 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30 (was 30), OpenFileDescriptor=312 (was 311) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7110 (was 7116) 2024-11-16T04:03:41,222 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30, OpenFileDescriptor=312, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7110 2024-11-16T04:03:41,223 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:41,223 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37391 2024-11-16T04:03:41,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32899b99, compressor=org.apache.hadoop.io.compress.GzipCodec@6dab86ab, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:41,224 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:41,224 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:41,225 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:41,250 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:41,250 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:41,263 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:41,276 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:41,276 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x40d25df3, L:/127.0.0.1:37391 - R:/127.0.0.1:42336] 2024-11-16T04:03:41,305 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:41,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:41,308 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37391 2024-11-16T04:03:41,312 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30 (was 30), OpenFileDescriptor=312 (was 312), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7104 (was 7110) 2024-11-16T04:03:41,317 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30, OpenFileDescriptor=312, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7104 2024-11-16T04:03:41,318 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:41,318 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42089 2024-11-16T04:03:41,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75f65c9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:41,319 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:41,319 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:41,320 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:41,348 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:41,349 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:41,362 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:41,374 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:41,375 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x08771097, L:/127.0.0.1:42089 - R:/127.0.0.1:40534] 2024-11-16T04:03:41,401 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:41,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:41,403 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42089 2024-11-16T04:03:41,409 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30 (was 30), OpenFileDescriptor=312 (was 312), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7098 (was 7104) 2024-11-16T04:03:41,416 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30, OpenFileDescriptor=312, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7097 2024-11-16T04:03:41,417 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:41,417 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46745 2024-11-16T04:03:41,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f394fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:41,417 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:41,418 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:41,418 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:41,445 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:41,445 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:41,458 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:41,471 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:41,471 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x87c22f4b, L:/127.0.0.1:46745 - R:/127.0.0.1:42008] 2024-11-16T04:03:41,499 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:41,500 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:41,500 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:42008 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:41,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:41,501 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:41,501 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46745 2024-11-16T04:03:41,506 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30 (was 30), OpenFileDescriptor=312 (was 312), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7091 (was 7097) 2024-11-16T04:03:41,511 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30, OpenFileDescriptor=312, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7090 2024-11-16T04:03:41,512 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:41,512 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33843 2024-11-16T04:03:41,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:41,513 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:41,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:41,514 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:41,544 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:41,544 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:41,559 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:41,571 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:41,572 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x4ff134bc, L:/127.0.0.1:33843 - R:/127.0.0.1:58846] 2024-11-16T04:03:41,599 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:41,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:41,600 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33843 2024-11-16T04:03:41,605 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30 (was 30), OpenFileDescriptor=312 (was 312), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7085 (was 7090) 2024-11-16T04:03:41,610 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30, OpenFileDescriptor=312, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7084 2024-11-16T04:03:41,611 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:41,611 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43577 2024-11-16T04:03:41,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b57e08a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:41,612 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:41,612 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:41,613 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:41,639 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:41,639 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:41,652 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:41,665 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:41,665 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x058f333a, L:/127.0.0.1:43577 - R:/127.0.0.1:46472] 2024-11-16T04:03:41,694 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:41,694 WARN [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:41,695 INFO [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-8c9b82cdb402f610ebb4880eba951da2-e956819777b09c4e-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:03:41,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:41,695 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:41,696 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43577 2024-11-16T04:03:41,701 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30 (was 30), OpenFileDescriptor=312 (was 312), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7078 (was 7084) 2024-11-16T04:03:41,705 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=30, OpenFileDescriptor=312, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7078 2024-11-16T04:03:41,706 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:41,707 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36333 2024-11-16T04:03:41,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c0639b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:41,707 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:41,708 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:41,708 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:41,734 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:41,735 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:41,750 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:41,763 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:41,763 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xde797fd6, L:/127.0.0.1:36333 - R:/127.0.0.1:44984] 2024-11-16T04:03:41,792 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:41,838 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36333 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=128ms, rpcTimeout=100ms 2024-11-16T04:03:41,838 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36333 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:03:41,838 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36333 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:03:41,838 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36333 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:03:41,838 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36333 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=128ms, rpcTimeout=100ms 2024-11-16T04:03:41,838 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36333 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=128ms, rpcTimeout=100ms 2024-11-16T04:03:41,838 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36333 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=128ms, rpcTimeout=100ms 2024-11-16T04:03:41,838 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36333 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=128ms, rpcTimeout=100ms 2024-11-16T04:03:41,838 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36333 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=128ms, rpcTimeout=100ms 2024-11-16T04:03:41,838 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36333 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:03:41,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:41,839 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36333 2024-11-16T04:03:41,847 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[3: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=false, clientTlsEnabled=true] Thread=31 (was 30) - Thread LEAK? -, OpenFileDescriptor=312 (was 312), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7071 (was 7078) 2024-11-16T04:03:41,853 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=31, OpenFileDescriptor=312, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7070 2024-11-16T04:03:41,854 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:41,855 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36763 2024-11-16T04:03:41,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b978cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:41,855 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:41,856 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:41,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:41,856 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:36763 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:03:41,856 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36763 2024-11-16T04:03:41,863 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=31 (was 31), OpenFileDescriptor=312 (was 312), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7070 (was 7070) 2024-11-16T04:03:41,870 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=31, OpenFileDescriptor=312, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7069 2024-11-16T04:03:41,871 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:41,872 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40693 2024-11-16T04:03:41,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d04b6d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:41,872 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:41,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:41,873 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:41,898 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:41,899 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:41,912 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:41,925 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:41,925 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x18bd9bb0, L:/127.0.0.1:40693 - R:/127.0.0.1:45710] 2024-11-16T04:03:41,953 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:45710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:42,007 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40693 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:03:42,116 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40693 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:42,226 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40693 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:42,337 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40693 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:42,446 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40693 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:42,557 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40693 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:42,667 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40693 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:42,777 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40693 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:42,794 WARN [TestNettyTlsIPC-1-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:42,794 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:44984 deadline: 1731729821893 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:44984 2024-11-16T04:03:42,794 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:44984 deadline: 1731729821894 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:44984 2024-11-16T04:03:42,795 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:44984 deadline: 1731729821894 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:44984 2024-11-16T04:03:42,795 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:44984 deadline: 1731729821894 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:44984 2024-11-16T04:03:42,795 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:44984 deadline: 1731729821895 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:44984 2024-11-16T04:03:42,796 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:44984 deadline: 1731729821895 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:44984 2024-11-16T04:03:42,796 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:44984 deadline: 1731729821895 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:44984 2024-11-16T04:03:42,796 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:44984 deadline: 1731729821895 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:44984 2024-11-16T04:03:42,796 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:44984 deadline: 1731729821896 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:44984 2024-11-16T04:03:42,887 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40693 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:42,955 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:45710 deadline: 1731729822108 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:45710 2024-11-16T04:03:42,955 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:45710 deadline: 1731729822217 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:45710 2024-11-16T04:03:42,956 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:03:42,956 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:45710 deadline: 1731729822327 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:45710 2024-11-16T04:03:42,957 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:45710 deadline: 1731729822438 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:45710 2024-11-16T04:03:42,957 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:45710 deadline: 1731729822547 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:45710 2024-11-16T04:03:42,958 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:45710 deadline: 1731729822659 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:45710 2024-11-16T04:03:42,958 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:45710 deadline: 1731729822769 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:45710 2024-11-16T04:03:42,959 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:45710 deadline: 1731729822880 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:45710 2024-11-16T04:03:42,996 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40693 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:42,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:42,997 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40693 2024-11-16T04:03:43,008 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=31 (was 31), OpenFileDescriptor=312 (was 312), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7061 (was 7069) 2024-11-16T04:03:43,014 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=31, OpenFileDescriptor=312, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7060 2024-11-16T04:03:43,015 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:43,016 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35891 2024-11-16T04:03:43,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d61e928, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:43,016 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:43,016 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:43,017 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:43,046 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:43,047 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:43,061 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:43,074 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:43,075 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x5c2a3d01, L:/127.0.0.1:35891 - R:/127.0.0.1:33980] 2024-11-16T04:03:43,104 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:43,105 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:43,105 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:33980 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:43,106 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:43,107 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:43,107 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:43,107 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:43,112 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32 (was 31) - Thread LEAK? -, OpenFileDescriptor=313 (was 312) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7055 (was 7060) 2024-11-16T04:03:43,116 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32, OpenFileDescriptor=313, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7055 2024-11-16T04:03:43,117 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:43,118 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36077 2024-11-16T04:03:43,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1153ca0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:43,118 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:43,119 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:43,119 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:43,147 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:43,147 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:43,160 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:43,185 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:43,185 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x6d847fc3, L:/127.0.0.1:36077 - R:/127.0.0.1:60288] 2024-11-16T04:03:43,215 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:43,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:43,227 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36077 2024-11-16T04:03:43,233 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32 (was 32), OpenFileDescriptor=313 (was 313), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7048 (was 7055) 2024-11-16T04:03:43,237 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32, OpenFileDescriptor=313, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7047 2024-11-16T04:03:43,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f5307d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:43,239 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:43,240 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46855 2024-11-16T04:03:43,240 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:43,240 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:43,241 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:43,267 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:43,267 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:43,280 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:43,293 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:43,293 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x347e7470, L:/127.0.0.1:46855 - R:/127.0.0.1:37830] 2024-11-16T04:03:43,321 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:43,322 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:43,322 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:37830 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:43,323 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:43,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:43,323 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46855 2024-11-16T04:03:43,328 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32 (was 32), OpenFileDescriptor=313 (was 313), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7040 (was 7047) 2024-11-16T04:03:43,333 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32, OpenFileDescriptor=313, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7040 2024-11-16T04:03:43,335 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:43,335 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43017 2024-11-16T04:03:43,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c2eccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:43,335 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:43,336 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:43,336 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:43,367 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:43,367 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:43,381 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:43,394 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:43,394 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xb6178395, L:/127.0.0.1:43017 - R:/127.0.0.1:41898] 2024-11-16T04:03:43,423 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:43,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:43,428 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43017 2024-11-16T04:03:43,433 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32 (was 32), OpenFileDescriptor=313 (was 313), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7034 (was 7040) 2024-11-16T04:03:43,438 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32, OpenFileDescriptor=313, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7033 2024-11-16T04:03:43,439 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:43,440 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39503 2024-11-16T04:03:43,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a5e4d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:43,440 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:43,440 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:43,441 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:43,468 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:43,468 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:43,481 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:43,494 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:43,494 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x9552fc17, L:/127.0.0.1:39503 - R:/127.0.0.1:41412] 2024-11-16T04:03:43,522 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:43,522 WARN [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:41412; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:43,523 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:43,523 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:39503 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:03:43,523 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39503 2024-11-16T04:03:43,528 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32 (was 32), OpenFileDescriptor=313 (was 313), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7027 (was 7033) 2024-11-16T04:03:43,533 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32, OpenFileDescriptor=313, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7027 2024-11-16T04:03:43,535 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:43,535 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35909 2024-11-16T04:03:43,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2395529d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:43,535 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:43,536 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:43,537 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:43,569 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:43,569 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:43,583 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:43,597 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:43,597 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x494b686f, L:/127.0.0.1:35909 - R:/127.0.0.1:39812] 2024-11-16T04:03:43,624 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:03:43,625 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:03:43,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:43,630 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32 (was 32), OpenFileDescriptor=314 (was 313) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7020 (was 7027) 2024-11-16T04:03:43,634 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32, OpenFileDescriptor=314, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7020 2024-11-16T04:03:43,635 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:43,635 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36577 2024-11-16T04:03:43,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b3b71ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:43,636 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:43,636 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:43,636 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:43,666 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:43,667 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:43,681 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:43,694 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:43,695 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xf49a36ce, L:/127.0.0.1:36577 - R:/127.0.0.1:54600] 2024-11-16T04:03:43,722 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:03:43,722 WARN [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:54600 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:54600 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:43,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:43,728 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32 (was 32), OpenFileDescriptor=315 (was 314) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7015 (was 7020) 2024-11-16T04:03:43,733 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32, OpenFileDescriptor=315, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7015 2024-11-16T04:03:43,734 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:43,735 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34467 2024-11-16T04:03:43,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21e50da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:43,735 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:43,736 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:43,736 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:43,764 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:43,765 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:43,779 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:43,792 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:43,792 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x3be57c27, L:/127.0.0.1:34467 - R:/127.0.0.1:54714] 2024-11-16T04:03:43,820 WARN [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:54714 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:54714 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:43,821 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:43,821 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34467 2024-11-16T04:03:43,826 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32 (was 32), OpenFileDescriptor=315 (was 315), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7008 (was 7015) 2024-11-16T04:03:43,831 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32, OpenFileDescriptor=315, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7008 2024-11-16T04:03:43,832 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:43,832 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39077 2024-11-16T04:03:43,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24dc5edc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:43,833 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:43,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:43,834 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:43,863 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:43,863 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:43,877 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:43,890 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:43,890 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x7b5e3081, L:/127.0.0.1:39077 - R:/127.0.0.1:36670] 2024-11-16T04:03:43,917 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:43,960 WARN [TestNettyTlsIPC-1-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:44,020 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:44,021 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:44,022 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:44,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:44,031 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32 (was 32), OpenFileDescriptor=316 (was 315) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=7002 (was 7008) 2024-11-16T04:03:44,036 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32, OpenFileDescriptor=316, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=7002 2024-11-16T04:03:44,038 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:44,038 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35367 2024-11-16T04:03:44,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f1ecd3, compressor=org.apache.hadoop.io.compress.GzipCodec@483ff395, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:44,038 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:44,039 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:44,039 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:44,066 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:44,066 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:44,080 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:44,092 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:44,093 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xbbcb7ecf, L:/127.0.0.1:35367 - R:/127.0.0.1:37342] 2024-11-16T04:03:44,121 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:44,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:44,123 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35367 2024-11-16T04:03:44,127 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32 (was 32), OpenFileDescriptor=316 (was 316), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=6996 (was 7002) 2024-11-16T04:03:44,132 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32, OpenFileDescriptor=316, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=6995 2024-11-16T04:03:44,134 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:44,134 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40915 2024-11-16T04:03:44,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@620b7190, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:44,134 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:44,135 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:44,135 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:44,164 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:44,164 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:44,178 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:44,190 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:44,190 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x911e58a8, L:/127.0.0.1:40915 - R:/127.0.0.1:45666] 2024-11-16T04:03:44,218 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:45666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:44,219 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:44,219 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40915 2024-11-16T04:03:44,224 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32 (was 32), OpenFileDescriptor=316 (was 316), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=6990 (was 6995) 2024-11-16T04:03:44,229 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32, OpenFileDescriptor=316, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6990 2024-11-16T04:03:44,230 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:44,231 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33175 2024-11-16T04:03:44,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e70e045, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:44,231 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:44,231 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:44,232 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:44,261 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:44,261 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:44,274 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:44,287 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:44,287 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x9d59da70, L:/127.0.0.1:33175 - R:/127.0.0.1:44626] 2024-11-16T04:03:44,314 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:44,315 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:44,315 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:44626 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:44,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:44,316 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:44,316 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33175 2024-11-16T04:03:44,322 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32 (was 32), OpenFileDescriptor=316 (was 316), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6983 (was 6990) 2024-11-16T04:03:44,326 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32, OpenFileDescriptor=316, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6983 2024-11-16T04:03:44,327 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:44,328 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33375 2024-11-16T04:03:44,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:44,328 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:44,328 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:44,329 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:44,354 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:44,354 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:44,368 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:44,380 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:44,380 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xbd560999, L:/127.0.0.1:33375 - R:/127.0.0.1:59748] 2024-11-16T04:03:44,407 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:44,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:44,408 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33375 2024-11-16T04:03:44,412 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32 (was 32), OpenFileDescriptor=316 (was 316), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6977 (was 6983) 2024-11-16T04:03:44,417 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32, OpenFileDescriptor=316, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6976 2024-11-16T04:03:44,418 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:44,419 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39457 2024-11-16T04:03:44,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52305973, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:44,419 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:44,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:44,420 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:44,451 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:44,451 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:44,464 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:44,476 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:44,476 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xfc3ecabb, L:/127.0.0.1:39457 - R:/127.0.0.1:39282] 2024-11-16T04:03:44,504 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:44,504 WARN [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:44,505 INFO [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-b7eb263162450cef5aab5635b9cef326-4f781152613ef529-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:03:44,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:44,505 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:44,506 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39457 2024-11-16T04:03:44,510 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32 (was 32), OpenFileDescriptor=316 (was 316), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6970 (was 6976) 2024-11-16T04:03:44,514 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=32, OpenFileDescriptor=316, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6970 2024-11-16T04:03:44,515 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:44,515 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44087 2024-11-16T04:03:44,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55fed69a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:44,516 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:44,516 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:44,516 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:44,562 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:44,563 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:44,576 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:44,589 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:44,589 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x57cdd545, L:/127.0.0.1:44087 - R:/127.0.0.1:34978] 2024-11-16T04:03:44,619 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:44,667 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44087 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=149ms, rpcTimeout=100ms 2024-11-16T04:03:44,668 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44087 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=150ms, rpcTimeout=100ms 2024-11-16T04:03:44,668 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44087 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=150ms, rpcTimeout=100ms 2024-11-16T04:03:44,668 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44087 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=149ms, rpcTimeout=100ms 2024-11-16T04:03:44,668 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44087 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=149ms, rpcTimeout=100ms 2024-11-16T04:03:44,668 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44087 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=150ms, rpcTimeout=100ms 2024-11-16T04:03:44,668 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44087 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=150ms, rpcTimeout=100ms 2024-11-16T04:03:44,668 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44087 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=150ms, rpcTimeout=100ms 2024-11-16T04:03:44,668 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44087 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=150ms, rpcTimeout=100ms 2024-11-16T04:03:44,668 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44087 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=150ms, rpcTimeout=100ms 2024-11-16T04:03:44,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:44,668 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44087 2024-11-16T04:03:44,675 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[4: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=true] Thread=33 (was 32) - Thread LEAK? -, OpenFileDescriptor=316 (was 316), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6958 (was 6970) 2024-11-16T04:03:44,681 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=33, OpenFileDescriptor=316, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6958 2024-11-16T04:03:44,682 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:44,682 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39467 2024-11-16T04:03:44,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@585c7d9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:44,683 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:44,683 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:44,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:44,684 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:39467 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:03:44,684 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39467 2024-11-16T04:03:44,691 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=33 (was 33), OpenFileDescriptor=316 (was 316), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6957 (was 6958) 2024-11-16T04:03:44,696 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=33, OpenFileDescriptor=316, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6957 2024-11-16T04:03:44,697 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:44,697 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44193 2024-11-16T04:03:44,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb3e209, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:44,697 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:44,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:44,712 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:44,724 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:44,724 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xad20c7e7, L:/127.0.0.1:44193 - R:/127.0.0.1:39232] 2024-11-16T04:03:44,725 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:44,806 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44193 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:03:44,917 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44193 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:45,026 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44193 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:45,137 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44193 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:45,247 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44193 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:45,356 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44193 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:45,466 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44193 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:45,577 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44193 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:45,620 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:34978 deadline: 1731729824719 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:34978 2024-11-16T04:03:45,620 WARN [TestNettyTlsIPC-1-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:45,620 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:34978 deadline: 1731729824720 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:34978 2024-11-16T04:03:45,620 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:34978 deadline: 1731729824720 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:34978 2024-11-16T04:03:45,620 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:34978 deadline: 1731729824720 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:34978 2024-11-16T04:03:45,620 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:34978 deadline: 1731729824720 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:34978 2024-11-16T04:03:45,621 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:34978 deadline: 1731729824720 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:34978 2024-11-16T04:03:45,621 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:34978 deadline: 1731729824720 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:34978 2024-11-16T04:03:45,621 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:34978 deadline: 1731729824721 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:34978 2024-11-16T04:03:45,621 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:34978 deadline: 1731729824721 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:34978 2024-11-16T04:03:45,686 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44193 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:03:45,726 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39232 deadline: 1731729824908 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39232 2024-11-16T04:03:45,726 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:03:45,726 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39232 deadline: 1731729825019 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39232 2024-11-16T04:03:45,726 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39232 deadline: 1731729825127 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39232 2024-11-16T04:03:45,727 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39232 deadline: 1731729825238 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39232 2024-11-16T04:03:45,727 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39232 deadline: 1731729825349 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39232 2024-11-16T04:03:45,727 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39232 deadline: 1731729825457 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39232 2024-11-16T04:03:45,727 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39232 deadline: 1731729825568 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39232 2024-11-16T04:03:45,727 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39232 deadline: 1731729825678 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39232 2024-11-16T04:03:45,796 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44193 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:45,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:45,796 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44193 2024-11-16T04:03:45,803 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=33 (was 33), OpenFileDescriptor=316 (was 316), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6952 (was 6957) 2024-11-16T04:03:45,809 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=33, OpenFileDescriptor=316, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6951 2024-11-16T04:03:45,811 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:45,811 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36847 2024-11-16T04:03:45,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff8584a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:45,812 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:45,812 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:45,829 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:45,843 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:45,844 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xe48cffe0, L:/127.0.0.1:36847 - R:/127.0.0.1:58536] 2024-11-16T04:03:45,844 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:45,845 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:45,845 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:58536 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:45,846 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:45,847 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:45,847 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:45,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:45,852 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=34 (was 33) - Thread LEAK? -, OpenFileDescriptor=317 (was 316) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6950 (was 6951) 2024-11-16T04:03:45,857 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=34, OpenFileDescriptor=317, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6950 2024-11-16T04:03:45,858 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:45,858 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43425 2024-11-16T04:03:45,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@470124e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:45,859 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:45,859 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:45,875 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:45,892 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:45,892 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x50aa99db, L:/127.0.0.1:43425 - R:/127.0.0.1:44918] 2024-11-16T04:03:45,893 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:45,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:45,902 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43425 2024-11-16T04:03:45,907 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=34 (was 34), OpenFileDescriptor=317 (was 317), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6952 (was 6950) - AvailableMemoryMB LEAK? - 2024-11-16T04:03:45,913 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=34, OpenFileDescriptor=317, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6951 2024-11-16T04:03:45,914 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e2468f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:45,914 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:45,915 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40081 2024-11-16T04:03:45,915 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:45,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:45,935 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:45,952 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:45,952 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xe9dee0ba, L:/127.0.0.1:40081 - R:/127.0.0.1:50768] 2024-11-16T04:03:45,952 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:45,953 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:45,953 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:50768 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:45,954 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:45,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:45,954 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40081 2024-11-16T04:03:45,959 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=34 (was 34), OpenFileDescriptor=317 (was 317), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6950 (was 6951) 2024-11-16T04:03:45,965 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=34, OpenFileDescriptor=317, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6948 2024-11-16T04:03:45,966 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:45,966 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39107 2024-11-16T04:03:45,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d95ed74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:45,966 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:45,967 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:45,987 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:46,004 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:46,004 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x0fd75ad7, L:/127.0.0.1:39107 - R:/127.0.0.1:36820] 2024-11-16T04:03:46,005 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:46,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:46,009 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39107 2024-11-16T04:03:46,015 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=34 (was 34), OpenFileDescriptor=317 (was 317), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6949 (was 6948) - AvailableMemoryMB LEAK? - 2024-11-16T04:03:46,020 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=34, OpenFileDescriptor=317, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6946 2024-11-16T04:03:46,021 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:46,021 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34091 2024-11-16T04:03:46,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744dc498, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:46,022 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:46,022 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:46,041 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:46,054 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:46,054 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x151af3b3, L:/127.0.0.1:34091 - R:/127.0.0.1:51814] 2024-11-16T04:03:46,055 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:46,055 WARN [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:51814; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:46,056 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:46,056 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:34091 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:03:46,056 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34091 2024-11-16T04:03:46,061 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=34 (was 34), OpenFileDescriptor=317 (was 317), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6949 (was 6946) - AvailableMemoryMB LEAK? - 2024-11-16T04:03:46,066 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=34, OpenFileDescriptor=317, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6949 2024-11-16T04:03:46,067 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:46,068 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44491 2024-11-16T04:03:46,068 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3598f8d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:46,068 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:46,068 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:46,083 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:46,096 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:46,096 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x1e3c3e0d, L:/127.0.0.1:44491 - R:/127.0.0.1:39792] 2024-11-16T04:03:46,096 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:03:46,096 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:03:46,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:46,101 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=34 (was 34), OpenFileDescriptor=318 (was 317) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6949 (was 6949) 2024-11-16T04:03:46,105 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=34, OpenFileDescriptor=318, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6949 2024-11-16T04:03:46,106 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:46,106 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37577 2024-11-16T04:03:46,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbe3a1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:46,107 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:46,107 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:46,123 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:46,137 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:46,137 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xb5ca130f, L:/127.0.0.1:37577 - R:/127.0.0.1:45612] 2024-11-16T04:03:46,137 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:03:46,137 WARN [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:45612 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:45612 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:46,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:46,142 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=34 (was 34), OpenFileDescriptor=319 (was 318) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6948 (was 6949) 2024-11-16T04:03:46,146 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=34, OpenFileDescriptor=319, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6948 2024-11-16T04:03:46,147 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:46,147 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40273 2024-11-16T04:03:46,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fee68ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:46,148 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:46,148 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:46,164 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:46,177 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:46,177 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x2385efac, L:/127.0.0.1:40273 - R:/127.0.0.1:58966] 2024-11-16T04:03:46,177 WARN [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:58966 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:58966 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:46,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:46,178 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40273 2024-11-16T04:03:46,182 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=34 (was 34), OpenFileDescriptor=319 (was 319), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6948 (was 6948) 2024-11-16T04:03:46,186 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=34, OpenFileDescriptor=319, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6948 2024-11-16T04:03:46,187 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:46,188 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42949 2024-11-16T04:03:46,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25e5d49b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:46,188 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:46,188 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:46,202 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:46,214 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:46,214 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xa53e12fa, L:/127.0.0.1:42949 - R:/127.0.0.1:41408] 2024-11-16T04:03:46,214 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:46,318 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:46,320 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:46,321 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:46,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:46,326 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=35 (was 34) - Thread LEAK? -, OpenFileDescriptor=320 (was 319) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6951 (was 6948) - AvailableMemoryMB LEAK? - 2024-11-16T04:03:46,331 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=35, OpenFileDescriptor=320, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6951 2024-11-16T04:03:46,332 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:46,333 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33527 2024-11-16T04:03:46,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@346b30a2, compressor=org.apache.hadoop.io.compress.GzipCodec@8e8ba96, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:46,333 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:46,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:46,348 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:46,361 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:46,361 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xe5f8d248, L:/127.0.0.1:33527 - R:/127.0.0.1:54476] 2024-11-16T04:03:46,361 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:46,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:46,363 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33527 2024-11-16T04:03:46,368 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=35 (was 35), OpenFileDescriptor=320 (was 320), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6951 (was 6951) 2024-11-16T04:03:46,372 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=35, OpenFileDescriptor=320, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6950 2024-11-16T04:03:46,373 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:46,373 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44399 2024-11-16T04:03:46,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@716cf2e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:46,374 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:46,374 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:46,390 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:46,402 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:46,402 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x6a0e3863, L:/127.0.0.1:44399 - R:/127.0.0.1:57780] 2024-11-16T04:03:46,403 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:46,404 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:46,404 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44399 2024-11-16T04:03:46,409 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=35 (was 35), OpenFileDescriptor=320 (was 320), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6950 (was 6950) 2024-11-16T04:03:46,413 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=35, OpenFileDescriptor=320, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6950 2024-11-16T04:03:46,414 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:46,414 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43061 2024-11-16T04:03:46,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c3edfd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:46,415 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:46,415 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:46,430 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:46,443 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:46,443 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x1f1918ee, L:/127.0.0.1:43061 - R:/127.0.0.1:57936] 2024-11-16T04:03:46,443 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:46,444 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:46,444 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:57936 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:46,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:46,445 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:46,445 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43061 2024-11-16T04:03:46,449 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=35 (was 35), OpenFileDescriptor=320 (was 320), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6949 (was 6950) 2024-11-16T04:03:46,453 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=35, OpenFileDescriptor=320, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6949 2024-11-16T04:03:46,454 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:46,455 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36881 2024-11-16T04:03:46,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:46,455 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:46,455 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:46,472 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:46,485 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:46,485 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x58584c08, L:/127.0.0.1:36881 - R:/127.0.0.1:47774] 2024-11-16T04:03:46,485 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:46,487 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:46,487 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36881 2024-11-16T04:03:46,491 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=35 (was 35), OpenFileDescriptor=320 (was 320), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6949 (was 6949) 2024-11-16T04:03:46,496 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=35, OpenFileDescriptor=320, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6949 2024-11-16T04:03:46,496 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:46,497 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45053 2024-11-16T04:03:46,497 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26542820, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:46,497 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:46,497 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:46,514 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:46,526 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:46,527 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x87be6310, L:/127.0.0.1:45053 - R:/127.0.0.1:34490] 2024-11-16T04:03:46,527 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:46,527 WARN [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:46,527 INFO [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-b6391f7cd66b188bf6b9fa797b9ccb82-0e7cecb98fd01b73-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:03:46,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:46,528 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:46,528 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45053 2024-11-16T04:03:46,532 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=35 (was 35), OpenFileDescriptor=320 (was 320), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6949 (was 6949) 2024-11-16T04:03:46,537 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=35, OpenFileDescriptor=320, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6949 2024-11-16T04:03:46,538 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:46,538 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39851 2024-11-16T04:03:46,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ef6e89e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:46,538 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:46,539 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:46,555 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:46,567 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:46,567 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x85eb6f60, L:/127.0.0.1:39851 - R:/127.0.0.1:59956] 2024-11-16T04:03:46,567 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:46,648 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39851 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=107ms, rpcTimeout=100ms 2024-11-16T04:03:46,648 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39851 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=107ms, rpcTimeout=100ms 2024-11-16T04:03:46,648 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39851 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:03:46,648 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39851 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:03:46,648 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39851 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=107ms, rpcTimeout=100ms 2024-11-16T04:03:46,648 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39851 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=107ms, rpcTimeout=100ms 2024-11-16T04:03:46,648 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39851 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=107ms, rpcTimeout=100ms 2024-11-16T04:03:46,648 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39851 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=107ms, rpcTimeout=100ms 2024-11-16T04:03:46,648 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39851 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:03:46,649 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39851 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:03:46,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:46,649 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39851 2024-11-16T04:03:46,659 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[5: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=RSA, keyPassword=[C@6d366c9b, acceptPlainText=true, clientTlsEnabled=false] Thread=36 (was 35) - Thread LEAK? -, OpenFileDescriptor=320 (was 320), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6949 (was 6949) 2024-11-16T04:03:46,665 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36, OpenFileDescriptor=320, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6949 2024-11-16T04:03:46,728 WARN [TestNettyTlsIPC-1-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:46,822 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:46,823 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33035 2024-11-16T04:03:46,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4af628fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:46,823 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:46,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:46,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:46,824 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:33035 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:03:46,824 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33035 2024-11-16T04:03:46,828 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=35 (was 36), OpenFileDescriptor=320 (was 320), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6948 (was 6949) 2024-11-16T04:03:46,833 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=35, OpenFileDescriptor=320, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6948 2024-11-16T04:03:46,834 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:46,834 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37631 2024-11-16T04:03:46,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a10f4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:46,835 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:46,835 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:46,835 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:46,861 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:46,861 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:46,874 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:46,887 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:46,887 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x646b4788, L:/127.0.0.1:37631 - R:/127.0.0.1:36524] 2024-11-16T04:03:46,931 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:46,966 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:37631 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=131ms, rpcTimeout=100ms 2024-11-16T04:03:47,076 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:37631 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:47,186 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:37631 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:47,296 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:37631 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:47,407 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:37631 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:47,517 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:37631 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:47,569 WARN [TestNettyTlsIPC-1-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:47,569 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:59956 deadline: 1731729826668 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:59956 2024-11-16T04:03:47,570 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:59956 deadline: 1731729826668 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:59956 2024-11-16T04:03:47,570 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:59956 deadline: 1731729826669 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:59956 2024-11-16T04:03:47,571 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:59956 deadline: 1731729826669 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:59956 2024-11-16T04:03:47,571 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:59956 deadline: 1731729826669 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:59956 2024-11-16T04:03:47,571 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:59956 deadline: 1731729826669 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:59956 2024-11-16T04:03:47,572 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:59956 deadline: 1731729826669 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:59956 2024-11-16T04:03:47,572 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:59956 deadline: 1731729826669 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:59956 2024-11-16T04:03:47,573 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:59956 deadline: 1731729826669 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:59956 2024-11-16T04:03:47,626 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:37631 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:47,737 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:37631 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:47,847 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:37631 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:47,932 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:36524 deadline: 1731729827067 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:36524 2024-11-16T04:03:47,933 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:03:47,933 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:36524 deadline: 1731729827177 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:36524 2024-11-16T04:03:47,933 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:36524 deadline: 1731729827287 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:36524 2024-11-16T04:03:47,933 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:36524 deadline: 1731729827397 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:36524 2024-11-16T04:03:47,933 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:36524 deadline: 1731729827508 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:36524 2024-11-16T04:03:47,933 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:36524 deadline: 1731729827619 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:36524 2024-11-16T04:03:47,934 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:36524 deadline: 1731729827728 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:36524 2024-11-16T04:03:47,934 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:36524 deadline: 1731729827839 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:36524 2024-11-16T04:03:47,956 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:37631 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:47,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:47,956 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37631 2024-11-16T04:03:47,967 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=35 (was 35), OpenFileDescriptor=320 (was 320), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6950 (was 6948) - AvailableMemoryMB LEAK? - 2024-11-16T04:03:47,971 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=35, OpenFileDescriptor=320, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6950 2024-11-16T04:03:47,972 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:47,972 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33239 2024-11-16T04:03:47,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64af2534, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:47,973 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:47,973 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:47,973 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:48,002 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:48,002 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:48,015 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:48,028 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:48,028 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xeade08f8, L:/127.0.0.1:33239 - R:/127.0.0.1:54258] 2024-11-16T04:03:48,057 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:48,058 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:48,059 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:54258 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:48,059 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:48,060 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:48,060 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:48,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:48,066 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36 (was 35) - Thread LEAK? -, OpenFileDescriptor=321 (was 320) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6950 (was 6950) 2024-11-16T04:03:48,070 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36, OpenFileDescriptor=321, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6950 2024-11-16T04:03:48,071 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:48,072 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34233 2024-11-16T04:03:48,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e9bcbae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:48,072 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:48,072 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:48,073 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:48,098 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:48,098 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:48,111 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:48,123 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:48,124 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x2e9dbe0a, L:/127.0.0.1:34233 - R:/127.0.0.1:48702] 2024-11-16T04:03:48,153 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:48,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:48,165 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34233 2024-11-16T04:03:48,170 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36 (was 36), OpenFileDescriptor=321 (was 321), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6949 (was 6950) 2024-11-16T04:03:48,175 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36, OpenFileDescriptor=321, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6949 2024-11-16T04:03:48,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300014f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:48,176 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:48,176 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38721 2024-11-16T04:03:48,176 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:48,176 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:48,177 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:48,201 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:48,201 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:48,214 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:48,226 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:48,226 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xb8b797d3, L:/127.0.0.1:38721 - R:/127.0.0.1:39990] 2024-11-16T04:03:48,257 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:48,257 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:48,258 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:39990 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:48,258 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:48,258 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:48,259 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38721 2024-11-16T04:03:48,263 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36 (was 36), OpenFileDescriptor=321 (was 321), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6949 (was 6949) 2024-11-16T04:03:48,268 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36, OpenFileDescriptor=321, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6949 2024-11-16T04:03:48,269 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:48,269 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43101 2024-11-16T04:03:48,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@132e5b37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:48,269 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:48,270 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:48,270 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:48,295 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:48,296 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:48,309 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:48,322 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:48,322 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xbc295cb1, L:/127.0.0.1:43101 - R:/127.0.0.1:47782] 2024-11-16T04:03:48,353 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:48,357 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:48,357 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43101 2024-11-16T04:03:48,363 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36 (was 36), OpenFileDescriptor=321 (was 321), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6944 (was 6949) 2024-11-16T04:03:48,369 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36, OpenFileDescriptor=321, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6944 2024-11-16T04:03:48,370 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:48,370 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40139 2024-11-16T04:03:48,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1296c3c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:48,370 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:48,371 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:48,371 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:48,396 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:48,396 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:48,409 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:48,422 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:48,422 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x5787f52c, L:/127.0.0.1:40139 - R:/127.0.0.1:53848] 2024-11-16T04:03:48,453 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:48,453 WARN [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:53848; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:48,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:48,454 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:40139 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:03:48,454 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40139 2024-11-16T04:03:48,459 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36 (was 36), OpenFileDescriptor=321 (was 321), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6942 (was 6944) 2024-11-16T04:03:48,464 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36, OpenFileDescriptor=321, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6942 2024-11-16T04:03:48,466 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:48,466 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38703 2024-11-16T04:03:48,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7319a60c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:48,466 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:48,467 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:48,467 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:48,492 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:48,493 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:48,506 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:48,519 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:48,519 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xb3601819, L:/127.0.0.1:38703 - R:/127.0.0.1:42224] 2024-11-16T04:03:48,548 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:03:48,548 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:03:48,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:48,553 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36 (was 36), OpenFileDescriptor=322 (was 321) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6942 (was 6942) 2024-11-16T04:03:48,557 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36, OpenFileDescriptor=322, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6942 2024-11-16T04:03:48,558 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:48,558 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41857 2024-11-16T04:03:48,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@565888ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:48,559 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:48,559 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:48,559 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:48,586 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:48,586 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:48,600 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:48,613 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:48,613 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x633750cb, L:/127.0.0.1:41857 - R:/127.0.0.1:53220] 2024-11-16T04:03:48,640 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:03:48,640 WARN [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:53220 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:53220 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:48,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:48,645 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36 (was 36), OpenFileDescriptor=323 (was 322) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6941 (was 6942) 2024-11-16T04:03:48,650 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36, OpenFileDescriptor=323, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6941 2024-11-16T04:03:48,651 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:48,651 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46335 2024-11-16T04:03:48,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f84ec3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:48,651 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:48,652 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:48,652 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:48,677 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:48,677 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:48,691 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:48,703 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:48,703 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xa6f4b014, L:/127.0.0.1:46335 - R:/127.0.0.1:53016] 2024-11-16T04:03:48,733 WARN [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:53016 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:53016 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:48,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:48,734 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46335 2024-11-16T04:03:48,738 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36 (was 36), OpenFileDescriptor=323 (was 323), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6941 (was 6941) 2024-11-16T04:03:48,742 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36, OpenFileDescriptor=323, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6941 2024-11-16T04:03:48,743 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:48,743 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45949 2024-11-16T04:03:48,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b9b94f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:48,743 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:48,744 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:48,744 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:48,773 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:48,773 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:48,787 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:48,799 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:48,799 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x926e183b, L:/127.0.0.1:45949 - R:/127.0.0.1:45798] 2024-11-16T04:03:48,828 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:45798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:48,930 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:48,930 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:48,931 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:48,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:48,934 WARN [TestNettyTlsIPC-1-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:48,936 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=37 (was 36) - Thread LEAK? -, OpenFileDescriptor=324 (was 323) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6938 (was 6941) 2024-11-16T04:03:48,940 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36, OpenFileDescriptor=324, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6938 2024-11-16T04:03:48,941 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:48,941 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40631 2024-11-16T04:03:48,941 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f9808ca, compressor=org.apache.hadoop.io.compress.GzipCodec@1eaf28cc, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:48,942 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:48,942 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:48,942 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:48,967 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:48,967 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:48,981 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:48,993 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:48,994 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x15618098, L:/127.0.0.1:40631 - R:/127.0.0.1:32800] 2024-11-16T04:03:49,023 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:32800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:49,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:49,025 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40631 2024-11-16T04:03:49,030 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36 (was 36), OpenFileDescriptor=324 (was 324), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6936 (was 6938) 2024-11-16T04:03:49,035 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36, OpenFileDescriptor=324, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6936 2024-11-16T04:03:49,036 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:49,036 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42539 2024-11-16T04:03:49,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49eb9f99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:49,036 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:49,037 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:49,037 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:49,062 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:49,062 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:49,075 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:49,088 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:49,088 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x573ab08c, L:/127.0.0.1:42539 - R:/127.0.0.1:53380] 2024-11-16T04:03:49,118 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:49,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:49,120 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42539 2024-11-16T04:03:49,125 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36 (was 36), OpenFileDescriptor=324 (was 324), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6934 (was 6936) 2024-11-16T04:03:49,129 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36, OpenFileDescriptor=324, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=6934 2024-11-16T04:03:49,130 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:49,130 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38303 2024-11-16T04:03:49,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1319617a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:49,130 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:49,131 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:49,131 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:49,159 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:49,159 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:49,172 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:49,185 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:49,185 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x56b8b864, L:/127.0.0.1:38303 - R:/127.0.0.1:57374] 2024-11-16T04:03:49,214 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:49,215 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:49,215 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:57374 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:49,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:49,216 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:49,216 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38303 2024-11-16T04:03:49,222 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36 (was 36), OpenFileDescriptor=324 (was 324), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=6934 (was 6934) 2024-11-16T04:03:49,227 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36, OpenFileDescriptor=324, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6934 2024-11-16T04:03:49,227 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:49,228 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44761 2024-11-16T04:03:49,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:49,228 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:49,228 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:49,229 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:49,253 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:49,253 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:49,266 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:49,283 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:49,284 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xba0d9d59, L:/127.0.0.1:44761 - R:/127.0.0.1:44814] 2024-11-16T04:03:49,313 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:49,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:49,314 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44761 2024-11-16T04:03:49,319 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36 (was 36), OpenFileDescriptor=324 (was 324), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6933 (was 6934) 2024-11-16T04:03:49,324 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36, OpenFileDescriptor=324, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6933 2024-11-16T04:03:49,325 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:49,325 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40581 2024-11-16T04:03:49,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@720320cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:49,325 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:49,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:49,326 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:49,352 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:49,352 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:49,365 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:49,378 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:49,378 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x0fd30539, L:/127.0.0.1:40581 - R:/127.0.0.1:56792] 2024-11-16T04:03:49,408 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:49,408 WARN [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:49,409 INFO [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-2e35e9fe133ea6efc284a7536f8ee317-d0fb53a938d5f5f7-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:03:49,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:49,409 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:49,410 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40581 2024-11-16T04:03:49,414 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36 (was 36), OpenFileDescriptor=324 (was 324), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6930 (was 6933) 2024-11-16T04:03:49,419 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=36, OpenFileDescriptor=324, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6930 2024-11-16T04:03:49,420 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:49,421 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46811 2024-11-16T04:03:49,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@753c9a88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:49,421 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:49,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:49,422 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:49,447 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:49,447 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:49,462 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:49,477 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:49,478 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x6a83cb2f, L:/127.0.0.1:46811 - R:/127.0.0.1:49250] 2024-11-16T04:03:49,509 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:49,557 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:46811 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:03:49,557 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:46811 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:03:49,557 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:46811 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:03:49,557 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:46811 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:03:49,557 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:46811 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:03:49,558 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:46811 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:03:49,558 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:46811 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:03:49,558 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:46811 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:03:49,558 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:46811 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:03:49,558 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:46811 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:03:49,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:49,558 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46811 2024-11-16T04:03:49,565 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[6: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=false, clientTlsEnabled=true] Thread=37 (was 36) - Thread LEAK? -, OpenFileDescriptor=324 (was 324), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6929 (was 6930) 2024-11-16T04:03:49,572 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=37, OpenFileDescriptor=324, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6929 2024-11-16T04:03:49,573 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:49,574 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38161 2024-11-16T04:03:49,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2554f5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:49,574 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:49,575 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:49,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:49,575 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:38161 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:03:49,576 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38161 2024-11-16T04:03:49,583 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=37 (was 37), OpenFileDescriptor=324 (was 324), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6928 (was 6929) 2024-11-16T04:03:49,588 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=37, OpenFileDescriptor=324, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6928 2024-11-16T04:03:49,589 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:49,589 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41369 2024-11-16T04:03:49,590 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a9593d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:49,590 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:49,590 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:49,591 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:49,618 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:49,618 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:49,631 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:49,644 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:49,644 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xd2e911e1, L:/127.0.0.1:41369 - R:/127.0.0.1:39334] 2024-11-16T04:03:49,674 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:49,726 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:41369 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=136ms, rpcTimeout=100ms 2024-11-16T04:03:49,836 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:41369 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:49,946 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:41369 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:50,056 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:41369 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:50,166 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:41369 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:50,276 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:41369 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:50,386 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:41369 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:50,496 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:41369 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:50,510 WARN [TestNettyTlsIPC-1-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:50,510 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49250 deadline: 1731729829610 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49250 2024-11-16T04:03:50,511 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49250 deadline: 1731729829610 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49250 2024-11-16T04:03:50,511 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49250 deadline: 1731729829610 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49250 2024-11-16T04:03:50,511 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49250 deadline: 1731729829610 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49250 2024-11-16T04:03:50,511 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49250 deadline: 1731729829610 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49250 2024-11-16T04:03:50,512 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49250 deadline: 1731729829611 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49250 2024-11-16T04:03:50,512 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49250 deadline: 1731729829611 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49250 2024-11-16T04:03:50,512 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49250 deadline: 1731729829611 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49250 2024-11-16T04:03:50,512 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49250 deadline: 1731729829611 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49250 2024-11-16T04:03:50,606 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:41369 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:50,675 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39334 deadline: 1731729829827 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39334 2024-11-16T04:03:50,676 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39334 deadline: 1731729829937 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39334 2024-11-16T04:03:50,677 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:03:50,677 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39334 deadline: 1731729830047 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39334 2024-11-16T04:03:50,677 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39334 deadline: 1731729830157 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39334 2024-11-16T04:03:50,677 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39334 deadline: 1731729830268 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39334 2024-11-16T04:03:50,678 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39334 deadline: 1731729830378 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39334 2024-11-16T04:03:50,678 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39334 deadline: 1731729830488 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39334 2024-11-16T04:03:50,678 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39334 deadline: 1731729830598 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39334 2024-11-16T04:03:50,716 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:41369 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:50,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:50,716 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41369 2024-11-16T04:03:50,729 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=37 (was 37), OpenFileDescriptor=324 (was 324), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6925 (was 6928) 2024-11-16T04:03:50,733 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=37, OpenFileDescriptor=324, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6925 2024-11-16T04:03:50,734 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:50,734 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33437 2024-11-16T04:03:50,734 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72ef945a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:50,734 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:50,735 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:50,735 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:50,762 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:50,762 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:50,776 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:50,788 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:50,788 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x41ef048f, L:/127.0.0.1:33437 - R:/127.0.0.1:52972] 2024-11-16T04:03:50,816 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:52972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:50,817 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:50,817 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:52972 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:50,818 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:50,819 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:50,819 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:50,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:50,824 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38 (was 37) - Thread LEAK? -, OpenFileDescriptor=325 (was 324) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6924 (was 6925) 2024-11-16T04:03:50,829 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38, OpenFileDescriptor=325, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6923 2024-11-16T04:03:50,830 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:50,830 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34475 2024-11-16T04:03:50,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62fb8d56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:50,831 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:50,831 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:50,832 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:50,862 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:50,862 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:50,875 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:50,887 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:50,887 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x35c02285, L:/127.0.0.1:34475 - R:/127.0.0.1:36218] 2024-11-16T04:03:50,916 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:50,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:50,928 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34475 2024-11-16T04:03:50,933 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38 (was 38), OpenFileDescriptor=325 (was 325), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6923 (was 6923) 2024-11-16T04:03:50,938 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38, OpenFileDescriptor=325, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6923 2024-11-16T04:03:50,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@201a3ffb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:50,940 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:50,940 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36677 2024-11-16T04:03:50,940 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:50,941 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:50,941 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:50,972 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:50,972 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:50,985 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:50,997 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:50,997 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x8eb4be83, L:/127.0.0.1:36677 - R:/127.0.0.1:49000] 2024-11-16T04:03:51,025 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:51,026 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:51,026 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:49000 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:51,026 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:51,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:51,027 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36677 2024-11-16T04:03:51,031 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38 (was 38), OpenFileDescriptor=325 (was 325), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6922 (was 6923) 2024-11-16T04:03:51,036 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38, OpenFileDescriptor=325, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6922 2024-11-16T04:03:51,037 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:51,038 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45123 2024-11-16T04:03:51,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d1c7b13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:51,038 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:51,039 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:51,039 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:51,063 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:51,064 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:51,077 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:51,089 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:51,090 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x68db830d, L:/127.0.0.1:45123 - R:/127.0.0.1:33662] 2024-11-16T04:03:51,119 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:51,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:51,123 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45123 2024-11-16T04:03:51,128 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38 (was 38), OpenFileDescriptor=325 (was 325), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6922 (was 6922) 2024-11-16T04:03:51,132 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38, OpenFileDescriptor=325, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6922 2024-11-16T04:03:51,133 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:51,133 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42747 2024-11-16T04:03:51,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6ce4a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:51,134 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:51,134 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:51,134 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:51,158 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:51,158 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:51,173 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:51,185 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:51,185 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xf37173fa, L:/127.0.0.1:42747 - R:/127.0.0.1:33322] 2024-11-16T04:03:51,213 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:51,213 WARN [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:33322; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:51,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:51,214 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:42747 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:03:51,214 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42747 2024-11-16T04:03:51,219 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38 (was 38), OpenFileDescriptor=325 (was 325), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6921 (was 6922) 2024-11-16T04:03:51,223 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38, OpenFileDescriptor=325, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6921 2024-11-16T04:03:51,224 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:51,225 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40323 2024-11-16T04:03:51,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d1a987, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:51,225 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:51,225 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:51,226 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:51,254 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:51,255 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:51,268 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:51,281 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:51,281 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x17151755, L:/127.0.0.1:40323 - R:/127.0.0.1:59616] 2024-11-16T04:03:51,309 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:03:51,309 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:03:51,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:51,314 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38 (was 38), OpenFileDescriptor=326 (was 325) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6921 (was 6921) 2024-11-16T04:03:51,319 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38, OpenFileDescriptor=326, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6921 2024-11-16T04:03:51,320 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:51,321 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45671 2024-11-16T04:03:51,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6977cc0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:51,321 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:51,321 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:51,322 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:51,349 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:51,349 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:51,363 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:51,376 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:51,376 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x62637d0e, L:/127.0.0.1:45671 - R:/127.0.0.1:43298] 2024-11-16T04:03:51,403 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:03:51,404 WARN [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:43298 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:43298 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:51,404 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:51,409 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38 (was 38), OpenFileDescriptor=327 (was 326) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6920 (was 6921) 2024-11-16T04:03:51,414 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6920 2024-11-16T04:03:51,415 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:51,415 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41027 2024-11-16T04:03:51,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20ddbd5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:51,416 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:51,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:51,416 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:51,441 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:51,441 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:51,455 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:51,467 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:51,467 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xc37cf2b0, L:/127.0.0.1:41027 - R:/127.0.0.1:48082] 2024-11-16T04:03:51,494 WARN [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:48082 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:48082 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:51,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:51,495 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41027 2024-11-16T04:03:51,500 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38 (was 38), OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6919 (was 6920) 2024-11-16T04:03:51,504 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6919 2024-11-16T04:03:51,504 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:51,505 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37199 2024-11-16T04:03:51,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da35cb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:51,505 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:51,505 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:51,506 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:51,534 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:51,534 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:51,547 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:51,560 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:51,560 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xf4d7a7c0, L:/127.0.0.1:37199 - R:/127.0.0.1:53458] 2024-11-16T04:03:51,588 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:51,679 WARN [TestNettyTlsIPC-1-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:51,690 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:51,690 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:51,691 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:51,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:51,696 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38 (was 38), OpenFileDescriptor=328 (was 327) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6919 (was 6919) 2024-11-16T04:03:51,700 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38, OpenFileDescriptor=328, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6919 2024-11-16T04:03:51,701 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:51,701 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33135 2024-11-16T04:03:51,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70a56cc1, compressor=org.apache.hadoop.io.compress.GzipCodec@70518855, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:51,702 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:51,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:51,702 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:51,729 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:51,730 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:51,743 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:51,755 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:51,755 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x4625ad9e, L:/127.0.0.1:33135 - R:/127.0.0.1:56722] 2024-11-16T04:03:51,783 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:51,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:51,785 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33135 2024-11-16T04:03:51,790 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38 (was 38), OpenFileDescriptor=328 (was 328), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6918 (was 6919) 2024-11-16T04:03:51,794 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38, OpenFileDescriptor=328, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6918 2024-11-16T04:03:51,795 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:51,795 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45447 2024-11-16T04:03:51,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c90f16a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:51,796 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:51,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:51,796 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:51,823 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:51,824 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:51,838 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:51,850 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:51,850 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xf0c273cc, L:/127.0.0.1:45447 - R:/127.0.0.1:35170] 2024-11-16T04:03:51,878 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:51,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:51,879 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45447 2024-11-16T04:03:51,885 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38 (was 38), OpenFileDescriptor=328 (was 328), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6917 (was 6918) 2024-11-16T04:03:51,889 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38, OpenFileDescriptor=328, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6917 2024-11-16T04:03:51,890 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:51,890 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39013 2024-11-16T04:03:51,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e991253, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:51,890 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:51,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:51,891 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:51,919 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:51,919 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:51,933 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:51,945 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:51,945 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x3975f56a, L:/127.0.0.1:39013 - R:/127.0.0.1:40012] 2024-11-16T04:03:51,972 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:51,973 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:51,973 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:40012 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:51,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:51,974 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:51,974 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39013 2024-11-16T04:03:51,980 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38 (was 38), OpenFileDescriptor=328 (was 328), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6917 (was 6917) 2024-11-16T04:03:51,985 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38, OpenFileDescriptor=328, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6916 2024-11-16T04:03:51,986 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:51,986 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34731 2024-11-16T04:03:51,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:51,986 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:51,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:51,987 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:52,011 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:52,012 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:52,024 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:52,037 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:52,037 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x6b3249e6, L:/127.0.0.1:34731 - R:/127.0.0.1:46620] 2024-11-16T04:03:52,066 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:52,067 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:52,067 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34731 2024-11-16T04:03:52,072 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38 (was 38), OpenFileDescriptor=328 (was 328), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6915 (was 6916) 2024-11-16T04:03:52,077 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38, OpenFileDescriptor=328, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6915 2024-11-16T04:03:52,078 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:52,078 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39965 2024-11-16T04:03:52,078 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c2be64e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:52,078 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:52,079 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:52,079 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:52,105 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:52,105 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:52,118 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:52,131 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:52,131 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x3d2061f1, L:/127.0.0.1:39965 - R:/127.0.0.1:47474] 2024-11-16T04:03:52,159 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:52,159 WARN [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:52,160 INFO [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-3c2f422347ad019ac91e2d3c48d6ecee-6a1c384b7d4f042d-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:03:52,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:52,160 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:52,161 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39965 2024-11-16T04:03:52,165 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38 (was 38), OpenFileDescriptor=328 (was 328), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6914 (was 6915) 2024-11-16T04:03:52,170 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=38, OpenFileDescriptor=328, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6914 2024-11-16T04:03:52,170 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:52,171 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34491 2024-11-16T04:03:52,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54fa5936, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:52,171 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:52,171 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:52,172 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:52,199 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:52,199 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:52,212 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:52,225 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:52,225 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xa84ce49c, L:/127.0.0.1:34491 - R:/127.0.0.1:47906] 2024-11-16T04:03:52,255 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:52,307 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34491 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:03:52,307 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34491 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:03:52,307 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34491 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:03:52,307 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34491 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:03:52,307 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34491 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:03:52,307 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34491 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:03:52,307 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34491 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:03:52,307 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34491 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:03:52,307 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34491 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:03:52,307 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34491 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:03:52,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:52,308 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34491 2024-11-16T04:03:52,314 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[7: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=true] Thread=39 (was 38) - Thread LEAK? -, OpenFileDescriptor=328 (was 328), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6912 (was 6914) 2024-11-16T04:03:52,319 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=39, OpenFileDescriptor=328, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6912 2024-11-16T04:03:52,320 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:52,321 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38741 2024-11-16T04:03:52,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aa66075, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:52,321 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:52,321 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:52,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:52,322 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:38741 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:03:52,322 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38741 2024-11-16T04:03:52,326 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=39 (was 39), OpenFileDescriptor=328 (was 328), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6912 (was 6912) 2024-11-16T04:03:52,331 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=39, OpenFileDescriptor=328, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6912 2024-11-16T04:03:52,332 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:52,332 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45757 2024-11-16T04:03:52,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d8dd045, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:52,332 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:52,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:52,348 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:52,360 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:52,360 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xcb89ec6b, L:/127.0.0.1:45757 - R:/127.0.0.1:58906] 2024-11-16T04:03:52,360 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:52,436 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45757 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=104ms, rpcTimeout=100ms 2024-11-16T04:03:52,547 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45757 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:52,657 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45757 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:52,767 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45757 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:52,877 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45757 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:52,986 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45757 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:53,096 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45757 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:53,206 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45757 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:53,256 WARN [TestNettyTlsIPC-1-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:53,257 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:47906 deadline: 1731729832356 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:47906 2024-11-16T04:03:53,257 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:47906 deadline: 1731729832356 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:47906 2024-11-16T04:03:53,258 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:47906 deadline: 1731729832356 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:47906 2024-11-16T04:03:53,258 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:47906 deadline: 1731729832356 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:47906 2024-11-16T04:03:53,258 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:47906 deadline: 1731729832356 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:47906 2024-11-16T04:03:53,259 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:47906 deadline: 1731729832356 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:47906 2024-11-16T04:03:53,259 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:47906 deadline: 1731729832356 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:47906 2024-11-16T04:03:53,259 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:47906 deadline: 1731729832357 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:47906 2024-11-16T04:03:53,260 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:47906 deadline: 1731729832357 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:47906 2024-11-16T04:03:53,317 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45757 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:53,362 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:58906 deadline: 1731729832537 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:58906 2024-11-16T04:03:53,362 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:03:53,362 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:58906 deadline: 1731729832648 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:58906 2024-11-16T04:03:53,363 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:58906 deadline: 1731729832759 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:58906 2024-11-16T04:03:53,363 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:58906 deadline: 1731729832868 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:58906 2024-11-16T04:03:53,363 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:58906 deadline: 1731729832978 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:58906 2024-11-16T04:03:53,364 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:58906 deadline: 1731729833088 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:58906 2024-11-16T04:03:53,364 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:58906 deadline: 1731729833198 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:58906 2024-11-16T04:03:53,365 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:58906 deadline: 1731729833308 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:58906 2024-11-16T04:03:53,426 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45757 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:53,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:53,426 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45757 2024-11-16T04:03:53,434 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=39 (was 39), OpenFileDescriptor=328 (was 328), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6917 (was 6912) - AvailableMemoryMB LEAK? - 2024-11-16T04:03:53,440 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=39, OpenFileDescriptor=328, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6917 2024-11-16T04:03:53,441 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:53,441 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34449 2024-11-16T04:03:53,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@668f533c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:53,441 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:53,441 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:53,456 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:53,469 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:53,469 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x752384b1, L:/127.0.0.1:34449 - R:/127.0.0.1:60658] 2024-11-16T04:03:53,469 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:53,470 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:53,470 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:60658 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:53,471 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:53,472 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:53,472 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:53,472 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:53,477 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=40 (was 39) - Thread LEAK? -, OpenFileDescriptor=329 (was 328) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6917 (was 6917) 2024-11-16T04:03:53,481 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=40, OpenFileDescriptor=329, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6917 2024-11-16T04:03:53,483 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:53,483 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38285 2024-11-16T04:03:53,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39b0a8c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:53,483 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:53,484 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:53,501 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:53,513 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:53,513 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x427a3026, L:/127.0.0.1:38285 - R:/127.0.0.1:36798] 2024-11-16T04:03:53,513 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:53,521 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:53,522 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38285 2024-11-16T04:03:53,527 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=40 (was 40), OpenFileDescriptor=329 (was 329), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6916 (was 6917) 2024-11-16T04:03:53,532 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=40, OpenFileDescriptor=329, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6916 2024-11-16T04:03:53,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1159205b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:53,533 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:53,533 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40891 2024-11-16T04:03:53,534 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:53,534 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:53,553 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:53,566 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:53,566 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xb2eb129e, L:/127.0.0.1:40891 - R:/127.0.0.1:55514] 2024-11-16T04:03:53,566 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:53,567 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:53,567 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:55514 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:53,567 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:53,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:53,568 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40891 2024-11-16T04:03:53,572 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=40 (was 40), OpenFileDescriptor=329 (was 329), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6915 (was 6916) 2024-11-16T04:03:53,577 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=40, OpenFileDescriptor=329, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6915 2024-11-16T04:03:53,578 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:53,578 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44475 2024-11-16T04:03:53,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5d9f6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:53,579 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:53,579 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:53,597 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:53,610 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:53,610 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x2784b2b3, L:/127.0.0.1:44475 - R:/127.0.0.1:52364] 2024-11-16T04:03:53,610 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:52364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:53,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:53,613 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44475 2024-11-16T04:03:53,617 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=40 (was 40), OpenFileDescriptor=329 (was 329), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6916 (was 6915) - AvailableMemoryMB LEAK? - 2024-11-16T04:03:53,622 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=40, OpenFileDescriptor=329, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6916 2024-11-16T04:03:53,623 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:53,624 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34821 2024-11-16T04:03:53,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61537867, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:53,624 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:53,624 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:53,640 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:53,653 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:53,653 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x7007c4c5, L:/127.0.0.1:34821 - R:/127.0.0.1:47408] 2024-11-16T04:03:53,654 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:53,654 WARN [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:47408; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:53,654 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:53,655 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:34821 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:03:53,655 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34821 2024-11-16T04:03:53,659 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=40 (was 40), OpenFileDescriptor=329 (was 329), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6915 (was 6916) 2024-11-16T04:03:53,663 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=40, OpenFileDescriptor=329, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6915 2024-11-16T04:03:53,665 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:53,665 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43407 2024-11-16T04:03:53,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2669c0a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:53,665 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:53,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:53,682 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:53,694 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:53,695 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x7b183b4f, L:/127.0.0.1:43407 - R:/127.0.0.1:35582] 2024-11-16T04:03:53,695 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:03:53,695 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:03:53,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:53,700 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=40 (was 40), OpenFileDescriptor=330 (was 329) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6914 (was 6915) 2024-11-16T04:03:53,704 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=40, OpenFileDescriptor=330, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6914 2024-11-16T04:03:53,705 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:53,705 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44591 2024-11-16T04:03:53,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71b2fffa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:53,705 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:53,706 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:53,722 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:53,735 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:53,735 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xcc548c45, L:/127.0.0.1:44591 - R:/127.0.0.1:40676] 2024-11-16T04:03:53,735 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:03:53,735 WARN [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:40676 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:40676 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:53,736 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:53,740 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=40 (was 40), OpenFileDescriptor=331 (was 330) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6914 (was 6914) 2024-11-16T04:03:53,745 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=40, OpenFileDescriptor=331, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6913 2024-11-16T04:03:53,746 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:53,746 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36695 2024-11-16T04:03:53,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b772c36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:53,746 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:53,747 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:53,763 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:53,776 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:53,776 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xb66024ca, L:/127.0.0.1:36695 - R:/127.0.0.1:36836] 2024-11-16T04:03:53,777 WARN [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:36836 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:36836 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:53,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:53,777 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36695 2024-11-16T04:03:53,782 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=40 (was 40), OpenFileDescriptor=331 (was 331), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6913 (was 6913) 2024-11-16T04:03:53,786 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=40, OpenFileDescriptor=331, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6913 2024-11-16T04:03:53,787 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:53,787 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42005 2024-11-16T04:03:53,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6581911f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:53,788 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:53,788 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:53,803 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:53,815 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:53,816 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xa709fb89, L:/127.0.0.1:42005 - R:/127.0.0.1:46070] 2024-11-16T04:03:53,816 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:53,918 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:53,919 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:53,920 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:53,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:53,929 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=41 (was 40) - Thread LEAK? -, OpenFileDescriptor=332 (was 331) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6911 (was 6913) 2024-11-16T04:03:53,935 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=41, OpenFileDescriptor=332, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6911 2024-11-16T04:03:53,936 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:53,936 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34105 2024-11-16T04:03:53,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff8729d, compressor=org.apache.hadoop.io.compress.GzipCodec@15ea4c23, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:53,936 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:53,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:53,965 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:53,984 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:53,984 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xd2edb132, L:/127.0.0.1:34105 - R:/127.0.0.1:33298] 2024-11-16T04:03:53,984 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:53,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:53,986 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34105 2024-11-16T04:03:53,991 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=41 (was 41), OpenFileDescriptor=332 (was 332), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6910 (was 6911) 2024-11-16T04:03:53,996 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=41, OpenFileDescriptor=332, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6910 2024-11-16T04:03:53,997 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:53,997 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46105 2024-11-16T04:03:53,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@441e4ebd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:53,997 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:53,997 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:54,015 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:54,027 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:54,028 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x1e33128c, L:/127.0.0.1:46105 - R:/127.0.0.1:57788] 2024-11-16T04:03:54,028 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:54,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:54,029 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46105 2024-11-16T04:03:54,034 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=41 (was 41), OpenFileDescriptor=332 (was 332), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6910 (was 6910) 2024-11-16T04:03:54,038 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=41, OpenFileDescriptor=332, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6910 2024-11-16T04:03:54,039 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:54,040 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45425 2024-11-16T04:03:54,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e2e9fb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:54,040 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:54,040 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:54,056 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:54,069 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:54,069 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x70814639, L:/127.0.0.1:45425 - R:/127.0.0.1:41708] 2024-11-16T04:03:54,069 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:54,070 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:54,070 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:41708 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:54,070 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:54,071 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:54,071 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45425 2024-11-16T04:03:54,075 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=41 (was 41), OpenFileDescriptor=332 (was 332), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6910 (was 6910) 2024-11-16T04:03:54,080 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=41, OpenFileDescriptor=332, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6910 2024-11-16T04:03:54,081 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:54,081 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40255 2024-11-16T04:03:54,081 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:54,081 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:54,082 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:54,098 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:54,110 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:54,110 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x1e86dddb, L:/127.0.0.1:40255 - R:/127.0.0.1:54330] 2024-11-16T04:03:54,111 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:54,112 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:54,112 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40255 2024-11-16T04:03:54,116 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=41 (was 41), OpenFileDescriptor=332 (was 332), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6910 (was 6910) 2024-11-16T04:03:54,121 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=41, OpenFileDescriptor=332, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6910 2024-11-16T04:03:54,122 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:54,122 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41659 2024-11-16T04:03:54,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ea224da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:54,122 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:54,123 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:54,138 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:54,150 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:54,151 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x58b82332, L:/127.0.0.1:41659 - R:/127.0.0.1:46902] 2024-11-16T04:03:54,151 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46902, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:54,151 WARN [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:54,151 INFO [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-c820cc59dd5087f63ca1257cfbf8d092-5f6af8014d6f330f-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:03:54,152 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:54,152 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:54,152 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41659 2024-11-16T04:03:54,157 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=41 (was 41), OpenFileDescriptor=332 (was 332), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=6910 (was 6910) 2024-11-16T04:03:54,161 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=41, OpenFileDescriptor=332, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6910 2024-11-16T04:03:54,162 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:54,163 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39295 2024-11-16T04:03:54,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61e99f9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:54,163 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:54,164 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:54,179 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:54,191 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:54,192 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x37a9470a, L:/127.0.0.1:39295 - R:/127.0.0.1:43986] 2024-11-16T04:03:54,192 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:54,276 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39295 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=103ms, rpcTimeout=100ms 2024-11-16T04:03:54,276 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39295 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=102ms, rpcTimeout=100ms 2024-11-16T04:03:54,276 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39295 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=103ms, rpcTimeout=100ms 2024-11-16T04:03:54,276 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39295 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=103ms, rpcTimeout=100ms 2024-11-16T04:03:54,276 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39295 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=103ms, rpcTimeout=100ms 2024-11-16T04:03:54,277 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39295 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=103ms, rpcTimeout=100ms 2024-11-16T04:03:54,277 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39295 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=102ms, rpcTimeout=100ms 2024-11-16T04:03:54,277 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39295 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=111ms, rpcTimeout=100ms 2024-11-16T04:03:54,277 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39295 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=111ms, rpcTimeout=100ms 2024-11-16T04:03:54,277 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39295 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=111ms, rpcTimeout=100ms 2024-11-16T04:03:54,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:54,277 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39295 2024-11-16T04:03:54,285 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[8: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@5b58ed3c, acceptPlainText=true, clientTlsEnabled=false] Thread=42 (was 41) - Thread LEAK? -, OpenFileDescriptor=332 (was 332), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 126) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6910 (was 6910) 2024-11-16T04:03:54,291 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42, OpenFileDescriptor=332, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6910 2024-11-16T04:03:54,366 WARN [TestNettyTlsIPC-1-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:54,474 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:54,474 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34109 2024-11-16T04:03:54,475 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@732422ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:54,475 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:54,475 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:54,475 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:54,475 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:34109 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:03:54,476 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34109 2024-11-16T04:03:54,480 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=41 (was 42), OpenFileDescriptor=332 (was 332), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6909 (was 6910) 2024-11-16T04:03:54,484 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=41, OpenFileDescriptor=332, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6909 2024-11-16T04:03:54,485 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:54,486 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33213 2024-11-16T04:03:54,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6fc0e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:54,486 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:54,486 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:54,487 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:54,519 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:54,519 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:54,533 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:54,545 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:54,545 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x578917c1, L:/127.0.0.1:33213 - R:/127.0.0.1:51610] 2024-11-16T04:03:54,572 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:54,626 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33213 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=140ms, rpcTimeout=100ms 2024-11-16T04:03:54,736 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33213 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:54,846 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33213 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:54,957 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33213 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:55,066 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33213 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:55,176 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33213 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:55,193 WARN [TestNettyTlsIPC-1-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:55,193 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:43986 deadline: 1731729834293 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:43986 2024-11-16T04:03:55,193 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:43986 deadline: 1731729834293 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:43986 2024-11-16T04:03:55,194 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:43986 deadline: 1731729834293 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:43986 2024-11-16T04:03:55,194 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:43986 deadline: 1731729834293 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:43986 2024-11-16T04:03:55,194 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:43986 deadline: 1731729834293 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:43986 2024-11-16T04:03:55,194 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:43986 deadline: 1731729834293 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:43986 2024-11-16T04:03:55,194 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:43986 deadline: 1731729834293 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:43986 2024-11-16T04:03:55,194 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:43986 deadline: 1731729834293 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:43986 2024-11-16T04:03:55,194 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:43986 deadline: 1731729834293 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:43986 2024-11-16T04:03:55,286 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33213 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:55,396 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33213 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:55,506 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33213 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:55,573 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51610 deadline: 1731729834727 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51610 2024-11-16T04:03:55,574 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:03:55,574 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51610 deadline: 1731729834837 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51610 2024-11-16T04:03:55,574 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51610 deadline: 1731729834948 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51610 2024-11-16T04:03:55,574 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51610 deadline: 1731729835058 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51610 2024-11-16T04:03:55,575 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51610 deadline: 1731729835167 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51610 2024-11-16T04:03:55,575 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51610 deadline: 1731729835277 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51610 2024-11-16T04:03:55,575 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51610 deadline: 1731729835388 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51610 2024-11-16T04:03:55,576 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51610 deadline: 1731729835498 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51610 2024-11-16T04:03:55,616 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33213 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:03:55,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:55,616 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33213 2024-11-16T04:03:55,629 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=41 (was 41), OpenFileDescriptor=332 (was 332), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6908 (was 6909) 2024-11-16T04:03:55,633 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=41, OpenFileDescriptor=332, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6908 2024-11-16T04:03:55,634 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:55,634 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40805 2024-11-16T04:03:55,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302ba004, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:55,634 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:55,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:55,635 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:55,662 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:55,662 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:55,675 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:55,687 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:55,687 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xd71b62eb, L:/127.0.0.1:40805 - R:/127.0.0.1:40524] 2024-11-16T04:03:55,714 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:55,715 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:55,715 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:40524 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:55,716 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:55,716 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:55,716 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:55,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:55,721 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42 (was 41) - Thread LEAK? -, OpenFileDescriptor=333 (was 332) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6908 (was 6908) 2024-11-16T04:03:55,725 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42, OpenFileDescriptor=333, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6908 2024-11-16T04:03:55,726 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:55,727 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34263 2024-11-16T04:03:55,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65597cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:55,727 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:55,727 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:55,728 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:55,752 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:55,753 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:55,774 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:55,786 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:55,787 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x3c560286, L:/127.0.0.1:34263 - R:/127.0.0.1:37548] 2024-11-16T04:03:55,814 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:55,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:55,823 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34263 2024-11-16T04:03:55,828 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42 (was 42), OpenFileDescriptor=333 (was 333), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6907 (was 6908) 2024-11-16T04:03:55,834 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42, OpenFileDescriptor=333, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6907 2024-11-16T04:03:55,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@656a03df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:55,835 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:55,835 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:32835 2024-11-16T04:03:55,835 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:55,836 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:55,836 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:55,861 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:55,861 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:55,874 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:55,886 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:55,886 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x4ea64217, L:/127.0.0.1:32835 - R:/127.0.0.1:35862] 2024-11-16T04:03:55,913 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:55,914 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:55,915 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:35862 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:55,915 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:55,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:55,915 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:32835 2024-11-16T04:03:55,920 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42 (was 42), OpenFileDescriptor=333 (was 333), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6906 (was 6907) 2024-11-16T04:03:55,925 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42, OpenFileDescriptor=333, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6905 2024-11-16T04:03:55,926 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:55,926 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42211 2024-11-16T04:03:55,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f372a5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:55,926 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:55,927 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:55,927 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:55,951 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:55,952 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:55,965 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:55,977 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:55,977 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x1a25c515, L:/127.0.0.1:42211 - R:/127.0.0.1:41914] 2024-11-16T04:03:56,006 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41914, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:56,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:56,009 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42211 2024-11-16T04:03:56,014 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42 (was 42), OpenFileDescriptor=333 (was 333), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6905 (was 6905) 2024-11-16T04:03:56,019 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42, OpenFileDescriptor=333, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6905 2024-11-16T04:03:56,020 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:56,020 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33449 2024-11-16T04:03:56,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57314e5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:56,020 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:56,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:56,021 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:56,046 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:56,046 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:56,059 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:56,071 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:56,072 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xd58c3e14, L:/127.0.0.1:33449 - R:/127.0.0.1:59902] 2024-11-16T04:03:56,100 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59902, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:56,100 WARN [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:59902; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:56,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:56,101 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:33449 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:03:56,101 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33449 2024-11-16T04:03:56,106 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42 (was 42), OpenFileDescriptor=333 (was 333), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6905 (was 6905) 2024-11-16T04:03:56,110 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42, OpenFileDescriptor=333, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6904 2024-11-16T04:03:56,111 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:56,112 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46387 2024-11-16T04:03:56,112 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55c32a66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:56,112 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:56,112 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:56,113 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:56,141 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:56,141 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:56,153 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:56,165 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:56,166 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x8c1d4110, L:/127.0.0.1:46387 - R:/127.0.0.1:60520] 2024-11-16T04:03:56,193 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:03:56,193 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:03:56,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:56,198 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42 (was 42), OpenFileDescriptor=334 (was 333) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6904 (was 6904) 2024-11-16T04:03:56,203 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42, OpenFileDescriptor=334, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6904 2024-11-16T04:03:56,204 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:56,204 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41733 2024-11-16T04:03:56,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@697f54f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:56,204 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:56,205 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:56,205 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:56,233 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:56,233 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:56,246 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:56,258 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:56,258 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xf4663092, L:/127.0.0.1:41733 - R:/127.0.0.1:50774] 2024-11-16T04:03:56,285 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:03:56,285 WARN [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:50774 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:50774 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:56,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:56,291 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42 (was 42), OpenFileDescriptor=335 (was 334) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6904 (was 6904) 2024-11-16T04:03:56,295 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42, OpenFileDescriptor=335, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6904 2024-11-16T04:03:56,296 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:56,296 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35197 2024-11-16T04:03:56,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c095baf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:56,296 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:56,297 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:56,297 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:56,321 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:56,322 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:56,334 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:56,347 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:56,347 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xd46e1982, L:/127.0.0.1:35197 - R:/127.0.0.1:39032] 2024-11-16T04:03:56,374 WARN [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:39032 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:39032 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:56,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:56,375 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35197 2024-11-16T04:03:56,379 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42 (was 42), OpenFileDescriptor=335 (was 335), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6904 (was 6904) 2024-11-16T04:03:56,384 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42, OpenFileDescriptor=335, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6904 2024-11-16T04:03:56,385 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:56,386 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34093 2024-11-16T04:03:56,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17e4aaf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:56,386 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:56,386 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:56,387 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:56,417 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:56,417 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:56,430 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:56,442 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:56,442 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x9901dbbc, L:/127.0.0.1:34093 - R:/127.0.0.1:55760] 2024-11-16T04:03:56,470 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:56,573 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:56,573 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:56,574 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:56,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:56,577 WARN [TestNettyTlsIPC-1-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:56,585 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=43 (was 42) - Thread LEAK? -, OpenFileDescriptor=336 (was 335) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6903 (was 6904) 2024-11-16T04:03:56,589 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42, OpenFileDescriptor=336, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6903 2024-11-16T04:03:56,590 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:56,590 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36837 2024-11-16T04:03:56,590 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51a7a942, compressor=org.apache.hadoop.io.compress.GzipCodec@6fd46d49, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:56,590 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:56,591 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:56,591 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:56,617 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:56,617 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:56,632 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:56,643 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:56,644 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xa38c67a6, L:/127.0.0.1:36837 - R:/127.0.0.1:47508] 2024-11-16T04:03:56,673 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:56,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:56,674 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36837 2024-11-16T04:03:56,679 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42 (was 42), OpenFileDescriptor=336 (was 336), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6902 (was 6903) 2024-11-16T04:03:56,683 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42, OpenFileDescriptor=336, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6901 2024-11-16T04:03:56,684 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:56,684 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:32955 2024-11-16T04:03:56,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73388282, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:56,684 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:56,685 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:56,685 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:56,713 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:56,713 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:56,727 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:56,739 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:56,739 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x987f4c48, L:/127.0.0.1:32955 - R:/127.0.0.1:41400] 2024-11-16T04:03:56,766 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:56,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:56,767 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:32955 2024-11-16T04:03:56,771 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42 (was 42), OpenFileDescriptor=336 (was 336), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6900 (was 6901) 2024-11-16T04:03:56,776 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42, OpenFileDescriptor=336, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6900 2024-11-16T04:03:56,777 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:56,777 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38649 2024-11-16T04:03:56,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aa03855, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:56,777 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:56,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:56,778 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:56,806 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:56,806 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:56,819 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:56,831 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:56,832 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x74cc8014, L:/127.0.0.1:38649 - R:/127.0.0.1:37472] 2024-11-16T04:03:56,858 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:56,859 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:56,859 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:37472 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:56,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:56,860 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:56,860 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38649 2024-11-16T04:03:56,865 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42 (was 42), OpenFileDescriptor=336 (was 336), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6900 (was 6900) 2024-11-16T04:03:56,870 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42, OpenFileDescriptor=336, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6900 2024-11-16T04:03:56,871 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:56,871 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44571 2024-11-16T04:03:56,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:56,871 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:56,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:56,872 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:56,897 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:56,897 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:56,910 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:56,923 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:56,923 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x238a6ebb, L:/127.0.0.1:44571 - R:/127.0.0.1:52750] 2024-11-16T04:03:56,950 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:52750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:56,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:56,951 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44571 2024-11-16T04:03:56,956 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42 (was 42), OpenFileDescriptor=336 (was 336), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6899 (was 6900) 2024-11-16T04:03:56,961 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42, OpenFileDescriptor=336, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6899 2024-11-16T04:03:56,962 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:56,962 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46783 2024-11-16T04:03:56,962 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@563eaa57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:56,962 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:56,963 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:56,963 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:56,989 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:56,989 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:57,002 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:57,015 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:57,015 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x8928987f, L:/127.0.0.1:46783 - R:/127.0.0.1:34546] 2024-11-16T04:03:57,043 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:57,043 WARN [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:57,043 INFO [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-1b47be09414d56703d86fef4a7fe4b9f-f68228173811af46-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:03:57,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:57,044 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:57,044 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46783 2024-11-16T04:03:57,049 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42 (was 42), OpenFileDescriptor=336 (was 336), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6899 (was 6899) 2024-11-16T04:03:57,054 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=42, OpenFileDescriptor=336, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6898 2024-11-16T04:03:57,055 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:57,055 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43031 2024-11-16T04:03:57,056 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33c6b67e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:57,056 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:57,056 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:57,057 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:57,088 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:57,088 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:57,100 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:57,112 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:57,113 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x18ad42f3, L:/127.0.0.1:43031 - R:/127.0.0.1:42570] 2024-11-16T04:03:57,141 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:57,197 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:43031 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=139ms, rpcTimeout=100ms 2024-11-16T04:03:57,197 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:43031 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=139ms, rpcTimeout=100ms 2024-11-16T04:03:57,197 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:43031 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=139ms, rpcTimeout=100ms 2024-11-16T04:03:57,197 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:43031 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=139ms, rpcTimeout=100ms 2024-11-16T04:03:57,197 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:43031 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=139ms, rpcTimeout=100ms 2024-11-16T04:03:57,197 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:43031 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=139ms, rpcTimeout=100ms 2024-11-16T04:03:57,197 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:43031 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=139ms, rpcTimeout=100ms 2024-11-16T04:03:57,197 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:43031 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=139ms, rpcTimeout=100ms 2024-11-16T04:03:57,197 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:43031 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=139ms, rpcTimeout=100ms 2024-11-16T04:03:57,197 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:43031 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=139ms, rpcTimeout=100ms 2024-11-16T04:03:57,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:57,197 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43031 2024-11-16T04:03:57,202 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[9: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=false, clientTlsEnabled=true] Thread=43 (was 42) - Thread LEAK? -, OpenFileDescriptor=336 (was 336), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6897 (was 6898) 2024-11-16T04:03:57,207 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=43, OpenFileDescriptor=336, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6897 2024-11-16T04:03:57,208 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:57,208 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38879 2024-11-16T04:03:57,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df2a61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:57,209 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:57,209 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:57,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:57,209 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:38879 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:03:57,210 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38879 2024-11-16T04:03:57,214 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=43 (was 43), OpenFileDescriptor=336 (was 336), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6897 (was 6897) 2024-11-16T04:03:57,219 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=43, OpenFileDescriptor=336, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6896 2024-11-16T04:03:57,220 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:57,221 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34323 2024-11-16T04:03:57,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e82c0a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:57,221 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:57,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:57,222 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:57,247 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:57,247 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:57,259 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:57,272 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:57,272 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xd7d35eb9, L:/127.0.0.1:34323 - R:/127.0.0.1:38276] 2024-11-16T04:03:57,299 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:57,356 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34323 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:03:57,466 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34323 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:57,576 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34323 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:57,686 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34323 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:57,796 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34323 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:57,907 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34323 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:58,016 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34323 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:58,127 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34323 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:03:58,142 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42570 deadline: 1731729837242 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42570 2024-11-16T04:03:58,142 WARN [TestNettyTlsIPC-1-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:58,142 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42570 deadline: 1731729837242 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42570 2024-11-16T04:03:58,142 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42570 deadline: 1731729837242 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42570 2024-11-16T04:03:58,142 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42570 deadline: 1731729837242 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42570 2024-11-16T04:03:58,143 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42570 deadline: 1731729837242 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42570 2024-11-16T04:03:58,143 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42570 deadline: 1731729837242 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42570 2024-11-16T04:03:58,143 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42570 deadline: 1731729837242 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42570 2024-11-16T04:03:58,143 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42570 deadline: 1731729837242 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42570 2024-11-16T04:03:58,143 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42570 deadline: 1731729837243 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42570 2024-11-16T04:03:58,236 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34323 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:03:58,300 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:38276 deadline: 1731729837457 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:38276 2024-11-16T04:03:58,301 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:38276 deadline: 1731729837568 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:38276 2024-11-16T04:03:58,301 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:03:58,301 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:38276 deadline: 1731729837678 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:38276 2024-11-16T04:03:58,302 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:38276 deadline: 1731729837788 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:38276 2024-11-16T04:03:58,302 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:38276 deadline: 1731729837899 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:38276 2024-11-16T04:03:58,302 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:38276 deadline: 1731729838008 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:38276 2024-11-16T04:03:58,302 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:38276 deadline: 1731729838118 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:38276 2024-11-16T04:03:58,303 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:38276 deadline: 1731729838228 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:38276 2024-11-16T04:03:58,347 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34323 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:03:58,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:58,347 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34323 2024-11-16T04:03:58,362 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=43 (was 43), OpenFileDescriptor=336 (was 336), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6895 (was 6896) 2024-11-16T04:03:58,367 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=43, OpenFileDescriptor=336, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6895 2024-11-16T04:03:58,368 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:58,368 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33385 2024-11-16T04:03:58,368 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@786f7488, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:58,368 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:58,369 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:58,369 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:58,397 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:58,397 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:58,410 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:58,422 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:58,422 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x2de987ae, L:/127.0.0.1:33385 - R:/127.0.0.1:51874] 2024-11-16T04:03:58,449 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:58,450 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:58,450 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:51874 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:58,451 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:58,451 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:58,452 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:58,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:58,457 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44 (was 43) - Thread LEAK? -, OpenFileDescriptor=337 (was 336) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6895 (was 6895) 2024-11-16T04:03:58,463 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44, OpenFileDescriptor=337, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6895 2024-11-16T04:03:58,464 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:58,464 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45499 2024-11-16T04:03:58,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8a6c23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:58,465 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:58,465 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:58,465 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:58,491 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:58,491 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:58,504 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:58,517 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:58,517 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xe6561a74, L:/127.0.0.1:45499 - R:/127.0.0.1:42280] 2024-11-16T04:03:58,544 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:58,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:58,553 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45499 2024-11-16T04:03:58,558 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44 (was 44), OpenFileDescriptor=337 (was 337), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6894 (was 6895) 2024-11-16T04:03:58,563 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44, OpenFileDescriptor=337, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6893 2024-11-16T04:03:58,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b57fba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:58,565 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:58,565 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39799 2024-11-16T04:03:58,565 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:58,565 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:58,566 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:58,592 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:58,592 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:58,606 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:58,618 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:58,618 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x1fa0de45, L:/127.0.0.1:39799 - R:/127.0.0.1:47148] 2024-11-16T04:03:58,644 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:58,645 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:58,645 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:47148 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:58,646 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:58,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:58,646 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39799 2024-11-16T04:03:58,651 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44 (was 44), OpenFileDescriptor=337 (was 337), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6892 (was 6893) 2024-11-16T04:03:58,655 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44, OpenFileDescriptor=337, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6892 2024-11-16T04:03:58,656 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:58,656 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44911 2024-11-16T04:03:58,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5babced, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:58,657 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:58,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:58,657 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:58,684 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:58,684 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:58,697 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:58,710 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:58,710 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x7ef0a792, L:/127.0.0.1:44911 - R:/127.0.0.1:45608] 2024-11-16T04:03:58,739 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:45608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:58,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:58,742 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44911 2024-11-16T04:03:58,747 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44 (was 44), OpenFileDescriptor=337 (was 337), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6891 (was 6892) 2024-11-16T04:03:58,753 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44, OpenFileDescriptor=337, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6891 2024-11-16T04:03:58,754 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:58,754 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45615 2024-11-16T04:03:58,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@530b9bc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:58,755 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:58,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:58,756 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:58,780 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:58,780 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:58,792 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:58,805 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:58,805 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x7916c420, L:/127.0.0.1:45615 - R:/127.0.0.1:47562] 2024-11-16T04:03:58,833 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:58,833 WARN [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:47562; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:58,834 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:58,834 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:45615 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:03:58,834 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45615 2024-11-16T04:03:58,839 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44 (was 44), OpenFileDescriptor=337 (was 337), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6892 (was 6891) - AvailableMemoryMB LEAK? - 2024-11-16T04:03:58,844 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44, OpenFileDescriptor=337, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6892 2024-11-16T04:03:58,846 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:58,846 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33245 2024-11-16T04:03:58,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47697b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:58,847 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:58,847 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:58,847 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:58,875 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:58,876 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:58,888 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:58,900 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:58,900 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x176e99bc, L:/127.0.0.1:33245 - R:/127.0.0.1:51350] 2024-11-16T04:03:58,927 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:03:58,928 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:03:58,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:58,933 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44 (was 44), OpenFileDescriptor=338 (was 337) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6892 (was 6892) 2024-11-16T04:03:58,937 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44, OpenFileDescriptor=338, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6892 2024-11-16T04:03:58,938 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:58,939 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37615 2024-11-16T04:03:58,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@613cbf44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:58,939 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:58,940 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:03:58,940 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:58,969 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:58,969 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:58,981 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:58,993 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:58,993 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xd89dc3a4, L:/127.0.0.1:37615 - R:/127.0.0.1:37492] 2024-11-16T04:03:59,021 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:03:59,021 WARN [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:37492 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:37492 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:59,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:59,026 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44 (was 44), OpenFileDescriptor=339 (was 338) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6892 (was 6892) 2024-11-16T04:03:59,032 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44, OpenFileDescriptor=339, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6892 2024-11-16T04:03:59,033 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:59,033 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45401 2024-11-16T04:03:59,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@372b5949, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:59,033 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:59,034 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:59,034 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:59,058 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:59,059 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:59,073 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:59,085 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:59,085 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xb14ee24a, L:/127.0.0.1:45401 - R:/127.0.0.1:57450] 2024-11-16T04:03:59,111 WARN [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:57450 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:57450 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:59,112 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:59,112 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45401 2024-11-16T04:03:59,117 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44 (was 44), OpenFileDescriptor=339 (was 339), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=6892 (was 6892) 2024-11-16T04:03:59,123 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44, OpenFileDescriptor=339, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6892 2024-11-16T04:03:59,124 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:59,124 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45997 2024-11-16T04:03:59,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b1ec924, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:59,125 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:59,125 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:59,125 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:59,151 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:59,152 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:59,164 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:59,176 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:59,176 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xb3a21951, L:/127.0.0.1:45997 - R:/127.0.0.1:56004] 2024-11-16T04:03:59,202 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:59,303 WARN [TestNettyTlsIPC-1-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:03:59,304 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:59,305 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:59,305 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:03:59,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:59,314 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44 (was 44), OpenFileDescriptor=340 (was 339) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 132) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6891 (was 6892) 2024-11-16T04:03:59,321 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44, OpenFileDescriptor=340, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6890 2024-11-16T04:03:59,322 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:59,322 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37627 2024-11-16T04:03:59,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@112a901a, compressor=org.apache.hadoop.io.compress.GzipCodec@19d30e8c, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:59,323 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:59,323 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:59,323 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:59,353 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:59,353 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:59,368 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:59,380 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:59,381 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x9cce8627, L:/127.0.0.1:37627 - R:/127.0.0.1:51772] 2024-11-16T04:03:59,407 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:59,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:59,409 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37627 2024-11-16T04:03:59,414 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44 (was 44), OpenFileDescriptor=340 (was 340), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6889 (was 6890) 2024-11-16T04:03:59,418 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44, OpenFileDescriptor=340, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6889 2024-11-16T04:03:59,419 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:59,420 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33273 2024-11-16T04:03:59,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25bd6b66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:59,420 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:59,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:59,421 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:59,448 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:59,448 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:59,461 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:59,474 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:59,474 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x0efd39aa, L:/127.0.0.1:33273 - R:/127.0.0.1:46248] 2024-11-16T04:03:59,500 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:59,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:59,502 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33273 2024-11-16T04:03:59,506 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44 (was 44), OpenFileDescriptor=340 (was 340), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6888 (was 6889) 2024-11-16T04:03:59,511 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44, OpenFileDescriptor=340, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6888 2024-11-16T04:03:59,512 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:59,512 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44001 2024-11-16T04:03:59,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a49c331, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:59,512 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:59,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:59,513 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:59,539 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:59,539 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:59,552 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:59,564 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:59,564 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xcb8643e8, L:/127.0.0.1:44001 - R:/127.0.0.1:45258] 2024-11-16T04:03:59,599 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:45258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:59,599 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:03:59,600 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:45258 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:03:59,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:59,600 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:03:59,600 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44001 2024-11-16T04:03:59,607 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44 (was 44), OpenFileDescriptor=340 (was 340), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6886 (was 6888) 2024-11-16T04:03:59,612 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44, OpenFileDescriptor=340, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6886 2024-11-16T04:03:59,613 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:59,613 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34103 2024-11-16T04:03:59,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:59,614 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:59,614 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:59,614 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:59,639 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:59,639 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:59,651 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:59,664 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:59,664 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xb16f6e13, L:/127.0.0.1:34103 - R:/127.0.0.1:60046] 2024-11-16T04:03:59,691 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:59,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:59,693 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34103 2024-11-16T04:03:59,697 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44 (was 44), OpenFileDescriptor=340 (was 340), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6886 (was 6886) 2024-11-16T04:03:59,702 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44, OpenFileDescriptor=340, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6886 2024-11-16T04:03:59,703 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:59,703 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43415 2024-11-16T04:03:59,703 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f65866f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:59,703 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:59,704 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:59,704 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:59,729 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:59,729 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:59,742 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:59,754 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:59,754 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x27a326d4, L:/127.0.0.1:43415 - R:/127.0.0.1:59658] 2024-11-16T04:03:59,783 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:59,783 WARN [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:59,783 INFO [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-dbc235b6716fc05f582238522384647c-20383e183daf279d-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:03:59,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:59,783 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:03:59,784 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43415 2024-11-16T04:03:59,788 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44 (was 44), OpenFileDescriptor=340 (was 340), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6884 (was 6886) 2024-11-16T04:03:59,793 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=44, OpenFileDescriptor=340, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6884 2024-11-16T04:03:59,794 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:59,794 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40421 2024-11-16T04:03:59,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5de68e40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:59,794 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:59,795 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:59,795 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:59,819 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:59,819 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:03:59,832 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:59,845 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:59,845 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x8c6c5075, L:/127.0.0.1:40421 - R:/127.0.0.1:54758] 2024-11-16T04:03:59,875 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:03:59,928 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40421 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=132ms, rpcTimeout=100ms 2024-11-16T04:03:59,928 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40421 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=132ms, rpcTimeout=100ms 2024-11-16T04:03:59,928 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40421 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=132ms, rpcTimeout=100ms 2024-11-16T04:03:59,928 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40421 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=132ms, rpcTimeout=100ms 2024-11-16T04:03:59,928 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40421 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=132ms, rpcTimeout=100ms 2024-11-16T04:03:59,928 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40421 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=132ms, rpcTimeout=100ms 2024-11-16T04:03:59,928 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40421 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=132ms, rpcTimeout=100ms 2024-11-16T04:03:59,928 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40421 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=132ms, rpcTimeout=100ms 2024-11-16T04:03:59,928 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40421 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=132ms, rpcTimeout=100ms 2024-11-16T04:03:59,928 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40421 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=133ms, rpcTimeout=100ms 2024-11-16T04:03:59,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:59,928 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40421 2024-11-16T04:03:59,937 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[10: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=true] Thread=45 (was 44) - Thread LEAK? -, OpenFileDescriptor=340 (was 340), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6883 (was 6884) 2024-11-16T04:03:59,942 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=45, OpenFileDescriptor=340, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6882 2024-11-16T04:03:59,943 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:59,943 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46295 2024-11-16T04:03:59,943 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@629bd235, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:59,943 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:59,944 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:59,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:03:59,944 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:46295 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:03:59,944 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46295 2024-11-16T04:03:59,949 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=45 (was 45), OpenFileDescriptor=340 (was 340), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6882 (was 6882) 2024-11-16T04:03:59,954 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=45, OpenFileDescriptor=340, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6882 2024-11-16T04:03:59,955 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:03:59,955 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37413 2024-11-16T04:03:59,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42f52d1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:03:59,955 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:03:59,956 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:03:59,969 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:03:59,981 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:03:59,981 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xd4f1adb7, L:/127.0.0.1:37413 - R:/127.0.0.1:39580] 2024-11-16T04:03:59,982 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:00,067 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:37413 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:00,177 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:37413 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:00,286 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:37413 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:00,397 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:37413 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:00,506 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:37413 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:00,616 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:37413 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:00,726 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:37413 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:00,837 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:37413 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:00,876 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:54758 deadline: 1731729839975 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:54758 2024-11-16T04:04:00,876 WARN [TestNettyTlsIPC-1-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:00,876 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:54758 deadline: 1731729839975 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:54758 2024-11-16T04:04:00,877 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:54758 deadline: 1731729839975 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:54758 2024-11-16T04:04:00,877 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:54758 deadline: 1731729839975 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:54758 2024-11-16T04:04:00,878 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:54758 deadline: 1731729839976 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:54758 2024-11-16T04:04:00,878 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:54758 deadline: 1731729839976 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:54758 2024-11-16T04:04:00,878 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:54758 deadline: 1731729839976 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:54758 2024-11-16T04:04:00,879 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:54758 deadline: 1731729839976 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:54758 2024-11-16T04:04:00,879 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:54758 deadline: 1731729839976 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:54758 2024-11-16T04:04:00,947 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:37413 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:00,983 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39580 deadline: 1731729840168 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39580 2024-11-16T04:04:00,984 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39580 deadline: 1731729840278 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39580 2024-11-16T04:04:00,984 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39580 deadline: 1731729840387 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39580 2024-11-16T04:04:00,984 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:04:00,984 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39580 deadline: 1731729840498 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39580 2024-11-16T04:04:00,984 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39580 deadline: 1731729840607 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39580 2024-11-16T04:04:00,985 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39580 deadline: 1731729840717 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39580 2024-11-16T04:04:00,985 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39580 deadline: 1731729840827 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39580 2024-11-16T04:04:00,985 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:39580 deadline: 1731729840939 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:39580 2024-11-16T04:04:01,056 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:37413 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:01,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:01,057 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37413 2024-11-16T04:04:01,069 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=45 (was 45), OpenFileDescriptor=340 (was 340), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6880 (was 6882) 2024-11-16T04:04:01,077 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=45, OpenFileDescriptor=340, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6880 2024-11-16T04:04:01,078 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:01,078 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43005 2024-11-16T04:04:01,078 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dce9342, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:01,079 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:01,079 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:01,098 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:01,110 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:01,110 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xb522fecb, L:/127.0.0.1:43005 - R:/127.0.0.1:58870] 2024-11-16T04:04:01,111 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:01,111 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:01,111 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:58870 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:01,112 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:01,112 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:01,113 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:01,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:01,117 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=46 (was 45) - Thread LEAK? -, OpenFileDescriptor=341 (was 340) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6880 (was 6880) 2024-11-16T04:04:01,122 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=46, OpenFileDescriptor=341, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6880 2024-11-16T04:04:01,124 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:01,124 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37749 2024-11-16T04:04:01,124 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c13f8a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:01,124 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:01,125 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:01,140 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:01,152 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:01,152 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x36d6293a, L:/127.0.0.1:37749 - R:/127.0.0.1:33846] 2024-11-16T04:04:01,152 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:01,159 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:01,159 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37749 2024-11-16T04:04:01,164 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=46 (was 46), OpenFileDescriptor=341 (was 341), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6880 (was 6880) 2024-11-16T04:04:01,168 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=46, OpenFileDescriptor=341, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6880 2024-11-16T04:04:01,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ec402ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:01,169 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:01,170 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34363 2024-11-16T04:04:01,170 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:01,170 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:01,184 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:01,197 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:01,197 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xbb1bda25, L:/127.0.0.1:34363 - R:/127.0.0.1:41244] 2024-11-16T04:04:01,197 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:01,198 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:01,198 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:41244 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:01,198 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:01,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:01,198 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34363 2024-11-16T04:04:01,203 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=46 (was 46), OpenFileDescriptor=341 (was 341), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6880 (was 6880) 2024-11-16T04:04:01,208 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=46, OpenFileDescriptor=341, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6880 2024-11-16T04:04:01,209 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:01,209 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38207 2024-11-16T04:04:01,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3665db3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:01,209 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:01,210 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:01,227 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:01,239 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:01,239 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x14d4726f, L:/127.0.0.1:38207 - R:/127.0.0.1:43834] 2024-11-16T04:04:01,239 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:01,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:01,242 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38207 2024-11-16T04:04:01,247 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=46 (was 46), OpenFileDescriptor=341 (was 341), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6880 (was 6880) 2024-11-16T04:04:01,251 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=46, OpenFileDescriptor=341, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6880 2024-11-16T04:04:01,252 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:01,252 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37077 2024-11-16T04:04:01,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ebdacc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:01,253 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:01,253 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:01,270 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:01,282 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:01,282 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xd3020890, L:/127.0.0.1:37077 - R:/127.0.0.1:58274] 2024-11-16T04:04:01,282 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:01,282 WARN [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:58274; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:01,283 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:01,283 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:37077 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:04:01,283 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37077 2024-11-16T04:04:01,288 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=46 (was 46), OpenFileDescriptor=341 (was 341), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6880 (was 6880) 2024-11-16T04:04:01,292 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=46, OpenFileDescriptor=341, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6880 2024-11-16T04:04:01,293 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:01,293 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35081 2024-11-16T04:04:01,293 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43e76db9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:01,293 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:01,294 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:01,310 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:01,323 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:01,323 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x489be98d, L:/127.0.0.1:35081 - R:/127.0.0.1:36280] 2024-11-16T04:04:01,324 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:04:01,324 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:04:01,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:01,328 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=46 (was 46), OpenFileDescriptor=342 (was 341) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6879 (was 6880) 2024-11-16T04:04:01,333 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=46, OpenFileDescriptor=342, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6879 2024-11-16T04:04:01,334 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:01,334 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42691 2024-11-16T04:04:01,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ebea567, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:01,334 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:01,335 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:01,351 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:01,364 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:01,364 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x5c06d279, L:/127.0.0.1:42691 - R:/127.0.0.1:53688] 2024-11-16T04:04:01,364 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:04:01,364 WARN [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:53688 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:53688 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:01,365 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:01,369 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=46 (was 46), OpenFileDescriptor=343 (was 342) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6879 (was 6879) 2024-11-16T04:04:01,374 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=46, OpenFileDescriptor=343, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6879 2024-11-16T04:04:01,375 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:01,375 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36863 2024-11-16T04:04:01,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1842d5d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:01,375 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:01,375 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:01,391 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:01,403 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:01,403 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x002a2734, L:/127.0.0.1:36863 - R:/127.0.0.1:38942] 2024-11-16T04:04:01,403 WARN [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:38942 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:38942 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:01,404 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:01,404 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36863 2024-11-16T04:04:01,408 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=46 (was 46), OpenFileDescriptor=343 (was 343), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6879 (was 6879) 2024-11-16T04:04:01,413 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=46, OpenFileDescriptor=343, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6879 2024-11-16T04:04:01,414 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:01,415 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37203 2024-11-16T04:04:01,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e80183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:01,415 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:01,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:01,431 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:01,443 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:01,444 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x692fe5c7, L:/127.0.0.1:37203 - R:/127.0.0.1:34560] 2024-11-16T04:04:01,444 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:01,546 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:01,546 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:01,546 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:01,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:01,555 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=47 (was 46) - Thread LEAK? -, OpenFileDescriptor=344 (was 343) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6879 (was 6879) 2024-11-16T04:04:01,563 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=47, OpenFileDescriptor=344, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6879 2024-11-16T04:04:01,565 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:01,565 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35243 2024-11-16T04:04:01,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23ff8806, compressor=org.apache.hadoop.io.compress.GzipCodec@49ec79ca, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:01,566 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:01,566 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:01,579 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:01,592 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:01,592 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xbe84fd17, L:/127.0.0.1:35243 - R:/127.0.0.1:46356] 2024-11-16T04:04:01,592 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:01,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:01,594 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35243 2024-11-16T04:04:01,599 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=47 (was 47), OpenFileDescriptor=344 (was 344), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6879 (was 6879) 2024-11-16T04:04:01,603 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=47, OpenFileDescriptor=344, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6879 2024-11-16T04:04:01,604 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:01,604 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34299 2024-11-16T04:04:01,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e6e89d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:01,605 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:01,605 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:01,621 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:01,633 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:01,633 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xc08ff9de, L:/127.0.0.1:34299 - R:/127.0.0.1:51568] 2024-11-16T04:04:01,633 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:01,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:01,634 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34299 2024-11-16T04:04:01,639 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=47 (was 47), OpenFileDescriptor=344 (was 344), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6878 (was 6879) 2024-11-16T04:04:01,643 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=47, OpenFileDescriptor=344, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6878 2024-11-16T04:04:01,644 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:01,645 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45207 2024-11-16T04:04:01,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27455ee0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:01,645 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:01,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:01,662 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:01,674 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:01,674 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x15b63920, L:/127.0.0.1:45207 - R:/127.0.0.1:34552] 2024-11-16T04:04:01,675 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:01,675 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:01,675 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:34552 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:01,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:01,676 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:01,676 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45207 2024-11-16T04:04:01,681 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=47 (was 47), OpenFileDescriptor=344 (was 344), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6879 (was 6878) - AvailableMemoryMB LEAK? - 2024-11-16T04:04:01,685 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=47, OpenFileDescriptor=344, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6879 2024-11-16T04:04:01,685 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:01,686 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33493 2024-11-16T04:04:01,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:01,686 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:01,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:01,703 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:01,715 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:01,715 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x6196a921, L:/127.0.0.1:33493 - R:/127.0.0.1:44340] 2024-11-16T04:04:01,715 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:01,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:01,716 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33493 2024-11-16T04:04:01,721 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=47 (was 47), OpenFileDescriptor=344 (was 344), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6879 (was 6879) 2024-11-16T04:04:01,725 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=47, OpenFileDescriptor=344, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6879 2024-11-16T04:04:01,726 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:01,726 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38033 2024-11-16T04:04:01,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e1b2456, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:01,726 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:01,727 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:01,744 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:01,756 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:01,756 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x6e34f2d9, L:/127.0.0.1:38033 - R:/127.0.0.1:42454] 2024-11-16T04:04:01,756 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:01,756 WARN [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:01,756 INFO [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-4c8b81ccbee8a1cb3771f669d77aadd6-71ca8ee85e139345-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:04:01,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:01,757 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:01,757 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38033 2024-11-16T04:04:01,762 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=47 (was 47), OpenFileDescriptor=344 (was 344), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6879 (was 6879) 2024-11-16T04:04:01,766 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=47, OpenFileDescriptor=344, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6879 2024-11-16T04:04:01,767 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:01,767 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36919 2024-11-16T04:04:01,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46bc987c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:01,768 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:01,768 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:01,785 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:01,798 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:01,798 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xfe602d89, L:/127.0.0.1:36919 - R:/127.0.0.1:50550] 2024-11-16T04:04:01,798 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:01,877 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36919 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:04:01,877 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36919 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:04:01,877 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36919 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:04:01,877 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36919 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:01,877 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36919 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:04:01,877 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36919 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:04:01,877 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36919 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:04:01,878 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36919 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:04:01,878 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36919 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:04:01,878 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36919 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:04:01,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:01,878 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36919 2024-11-16T04:04:01,886 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[11: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=RSA, certKeyType=EC, keyPassword=[C@24faea88, acceptPlainText=true, clientTlsEnabled=false] Thread=48 (was 47) - Thread LEAK? -, OpenFileDescriptor=344 (was 344), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6879 (was 6879) 2024-11-16T04:04:01,893 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48, OpenFileDescriptor=344, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6879 2024-11-16T04:04:01,986 WARN [TestNettyTlsIPC-1-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:02,133 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:02,133 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35595 2024-11-16T04:04:02,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@599f094a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:02,133 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:02,134 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:02,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:02,134 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:35595 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:04:02,134 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35595 2024-11-16T04:04:02,139 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=47 (was 48), OpenFileDescriptor=344 (was 344), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6879 (was 6879) 2024-11-16T04:04:02,143 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=47, OpenFileDescriptor=344, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6878 2024-11-16T04:04:02,144 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:02,144 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45583 2024-11-16T04:04:02,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27364ce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:02,144 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:02,145 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:02,145 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:02,172 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:02,173 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:02,185 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:02,197 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:02,197 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xffa500e8, L:/127.0.0.1:45583 - R:/127.0.0.1:37492] 2024-11-16T04:04:02,226 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37492, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:02,276 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45583 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=131ms, rpcTimeout=100ms 2024-11-16T04:04:02,386 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45583 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:02,496 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45583 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:02,607 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45583 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:02,716 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45583 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:02,800 WARN [TestNettyTlsIPC-1-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:02,800 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:50550 deadline: 1731729841899 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:50550 2024-11-16T04:04:02,800 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:50550 deadline: 1731729841899 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:50550 2024-11-16T04:04:02,801 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:50550 deadline: 1731729841899 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:50550 2024-11-16T04:04:02,801 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:50550 deadline: 1731729841899 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:50550 2024-11-16T04:04:02,801 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:50550 deadline: 1731729841899 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:50550 2024-11-16T04:04:02,801 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:50550 deadline: 1731729841899 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:50550 2024-11-16T04:04:02,802 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:50550 deadline: 1731729841899 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:50550 2024-11-16T04:04:02,802 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:50550 deadline: 1731729841899 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:50550 2024-11-16T04:04:02,802 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:50550 deadline: 1731729841899 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:50550 2024-11-16T04:04:02,826 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45583 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:02,936 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45583 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:03,046 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45583 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:03,156 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45583 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:03,227 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:37492 deadline: 1731729842376 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:37492 2024-11-16T04:04:03,228 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:37492 deadline: 1731729842487 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:37492 2024-11-16T04:04:03,228 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:04:03,228 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:37492 deadline: 1731729842598 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:37492 2024-11-16T04:04:03,228 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:37492 deadline: 1731729842707 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:37492 2024-11-16T04:04:03,229 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:37492 deadline: 1731729842818 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:37492 2024-11-16T04:04:03,229 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:37492 deadline: 1731729842927 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:37492 2024-11-16T04:04:03,229 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:37492 deadline: 1731729843038 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:37492 2024-11-16T04:04:03,229 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:37492 deadline: 1731729843148 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:37492 2024-11-16T04:04:03,266 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45583 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:03,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:03,266 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45583 2024-11-16T04:04:03,277 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=47 (was 47), OpenFileDescriptor=344 (was 344), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6878 (was 6878) 2024-11-16T04:04:03,285 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=47, OpenFileDescriptor=344, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6878 2024-11-16T04:04:03,286 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:03,286 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34333 2024-11-16T04:04:03,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ccaa78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:03,287 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:03,287 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:03,288 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:03,316 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:03,316 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:03,329 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:03,341 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:03,341 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x94d2a38a, L:/127.0.0.1:34333 - R:/127.0.0.1:38092] 2024-11-16T04:04:03,367 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:03,368 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:03,368 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:38092 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:03,369 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:03,369 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:03,370 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:03,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:03,375 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48 (was 47) - Thread LEAK? -, OpenFileDescriptor=345 (was 344) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6878 (was 6878) 2024-11-16T04:04:03,379 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48, OpenFileDescriptor=345, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6878 2024-11-16T04:04:03,380 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:03,381 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45939 2024-11-16T04:04:03,381 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f744709, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:03,381 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:03,382 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:03,382 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:03,410 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:03,410 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:03,423 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:03,436 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:03,436 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x6f3af1c3, L:/127.0.0.1:45939 - R:/127.0.0.1:39990] 2024-11-16T04:04:03,462 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:03,471 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:03,471 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45939 2024-11-16T04:04:03,477 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48 (was 48), OpenFileDescriptor=345 (was 345), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6878 (was 6878) 2024-11-16T04:04:03,482 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48, OpenFileDescriptor=345, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6878 2024-11-16T04:04:03,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c44e92e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:03,483 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:03,484 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39803 2024-11-16T04:04:03,484 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:03,484 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:03,484 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:03,509 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:03,509 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:03,522 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:03,535 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:03,535 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xff0c3d85, L:/127.0.0.1:39803 - R:/127.0.0.1:45954] 2024-11-16T04:04:03,561 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:45954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:03,562 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:03,562 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:45954 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:03,563 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:03,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:03,563 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39803 2024-11-16T04:04:03,568 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48 (was 48), OpenFileDescriptor=345 (was 345), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6878 (was 6878) 2024-11-16T04:04:03,572 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48, OpenFileDescriptor=345, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6878 2024-11-16T04:04:03,573 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:03,573 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33013 2024-11-16T04:04:03,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e23f1e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:03,574 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:03,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:03,574 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:03,602 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:03,602 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:03,616 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:03,629 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:03,629 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x4998c5b1, L:/127.0.0.1:33013 - R:/127.0.0.1:56120] 2024-11-16T04:04:03,655 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:03,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:03,657 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33013 2024-11-16T04:04:03,662 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48 (was 48), OpenFileDescriptor=345 (was 345), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6878 (was 6878) 2024-11-16T04:04:03,667 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48, OpenFileDescriptor=345, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6877 2024-11-16T04:04:03,668 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:03,669 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38465 2024-11-16T04:04:03,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@671e3b3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:03,669 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:03,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:03,670 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:03,696 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:03,697 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:03,709 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:03,723 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:03,723 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x81ab0e2c, L:/127.0.0.1:38465 - R:/127.0.0.1:41346] 2024-11-16T04:04:03,750 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:03,750 WARN [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:41346; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:03,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:03,751 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:38465 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:04:03,751 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38465 2024-11-16T04:04:03,756 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48 (was 48), OpenFileDescriptor=345 (was 345), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6877 (was 6877) 2024-11-16T04:04:03,760 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48, OpenFileDescriptor=345, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6877 2024-11-16T04:04:03,762 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:03,762 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43067 2024-11-16T04:04:03,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36a5e975, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:03,762 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:03,763 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:03,763 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:03,789 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:03,790 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:03,802 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:03,815 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:03,815 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x1ee06d89, L:/127.0.0.1:43067 - R:/127.0.0.1:54214] 2024-11-16T04:04:03,844 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:04:03,844 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:04:03,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:03,849 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48 (was 48), OpenFileDescriptor=346 (was 345) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6878 (was 6877) - AvailableMemoryMB LEAK? - 2024-11-16T04:04:03,854 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6878 2024-11-16T04:04:03,855 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:03,855 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43995 2024-11-16T04:04:03,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05f65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:03,855 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:03,856 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:03,856 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:03,891 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:03,891 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:03,903 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:03,916 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:03,916 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x599922f3, L:/127.0.0.1:43995 - R:/127.0.0.1:35828] 2024-11-16T04:04:03,943 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:04:03,943 WARN [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:35828 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:35828 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:03,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:03,949 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48 (was 48), OpenFileDescriptor=347 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6877 (was 6878) 2024-11-16T04:04:03,953 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48, OpenFileDescriptor=347, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6877 2024-11-16T04:04:03,954 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:03,955 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45093 2024-11-16T04:04:03,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21fd5f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:03,955 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:03,955 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:03,956 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:03,984 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:03,984 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:03,997 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:04,010 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:04,010 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xb2f0496e, L:/127.0.0.1:45093 - R:/127.0.0.1:46152] 2024-11-16T04:04:04,036 WARN [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:46152 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:46152 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:04,037 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:04,037 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45093 2024-11-16T04:04:04,041 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48 (was 48), OpenFileDescriptor=347 (was 347), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6877 (was 6877) 2024-11-16T04:04:04,046 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48, OpenFileDescriptor=347, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6877 2024-11-16T04:04:04,047 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:04,048 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34865 2024-11-16T04:04:04,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e882465, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:04,048 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:04,048 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:04,049 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:04,077 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:04,077 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:04,091 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:04,103 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:04,104 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x3676fffa, L:/127.0.0.1:34865 - R:/127.0.0.1:57818] 2024-11-16T04:04:04,130 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:04,230 WARN [TestNettyTlsIPC-1-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:04,232 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:04,233 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:04,233 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:04,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:04,243 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48 (was 48), OpenFileDescriptor=348 (was 347) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6877 (was 6877) 2024-11-16T04:04:04,252 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48, OpenFileDescriptor=348, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6877 2024-11-16T04:04:04,254 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:04,254 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44931 2024-11-16T04:04:04,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76f9abc2, compressor=org.apache.hadoop.io.compress.GzipCodec@46c23637, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:04,254 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:04,255 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:04,255 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:04,285 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:04,285 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:04,298 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:04,312 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:04,312 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x74977e00, L:/127.0.0.1:44931 - R:/127.0.0.1:59538] 2024-11-16T04:04:04,337 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:04,339 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:04,339 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44931 2024-11-16T04:04:04,345 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48 (was 48), OpenFileDescriptor=348 (was 348), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6876 (was 6877) 2024-11-16T04:04:04,349 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48, OpenFileDescriptor=348, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6876 2024-11-16T04:04:04,350 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:04,350 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34431 2024-11-16T04:04:04,351 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3009303b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:04,351 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:04,351 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:04,351 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:04,383 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:04,384 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:04,397 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:04,409 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:04,410 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xa3dc7529, L:/127.0.0.1:34431 - R:/127.0.0.1:38860] 2024-11-16T04:04:04,437 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:04,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:04,438 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34431 2024-11-16T04:04:04,443 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48 (was 48), OpenFileDescriptor=348 (was 348), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6876 (was 6876) 2024-11-16T04:04:04,447 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48, OpenFileDescriptor=348, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6876 2024-11-16T04:04:04,448 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:04,448 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37211 2024-11-16T04:04:04,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b60b5c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:04,448 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:04,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:04,449 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:04,475 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:04,476 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:04,488 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:04,501 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:04,502 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x60fbd8e5, L:/127.0.0.1:37211 - R:/127.0.0.1:59560] 2024-11-16T04:04:04,528 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:04,529 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:04,529 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:59560 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:04,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:04,529 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:04,529 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37211 2024-11-16T04:04:04,535 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48 (was 48), OpenFileDescriptor=348 (was 348), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6876 (was 6876) 2024-11-16T04:04:04,540 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48, OpenFileDescriptor=348, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6876 2024-11-16T04:04:04,541 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:04,542 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41605 2024-11-16T04:04:04,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:04,542 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:04,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:04,543 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:04,573 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:04,573 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:04,587 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:04,600 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:04,600 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x55b0496b, L:/127.0.0.1:41605 - R:/127.0.0.1:38418] 2024-11-16T04:04:04,625 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:04,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:04,626 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41605 2024-11-16T04:04:04,631 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48 (was 48), OpenFileDescriptor=348 (was 348), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6876 (was 6876) 2024-11-16T04:04:04,635 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48, OpenFileDescriptor=348, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6876 2024-11-16T04:04:04,636 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:04,637 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46357 2024-11-16T04:04:04,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e63d652, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:04,637 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:04,637 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:04,638 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:04,667 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:04,667 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:04,680 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:04,693 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:04,693 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x535136eb, L:/127.0.0.1:46357 - R:/127.0.0.1:47442] 2024-11-16T04:04:04,719 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:04,719 WARN [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:04,719 INFO [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-b6730e5db4f2785aca510bcc682fd6b9-8575cbd3fecd58cd-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:04:04,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:04,720 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:04,720 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46357 2024-11-16T04:04:04,725 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48 (was 48), OpenFileDescriptor=348 (was 348), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6876 (was 6876) 2024-11-16T04:04:04,730 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=48, OpenFileDescriptor=348, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6876 2024-11-16T04:04:04,731 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:04,731 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:32871 2024-11-16T04:04:04,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b6fa40e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:04,731 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:04,732 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:04,732 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:04,762 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:04,762 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:04,775 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:04,788 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:04,788 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xfd8ba518, L:/127.0.0.1:32871 - R:/127.0.0.1:33774] 2024-11-16T04:04:04,814 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:04,867 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:32871 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:04:04,867 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:32871 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:04:04,867 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:32871 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:04:04,868 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:32871 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:04:04,868 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:32871 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:04:04,868 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:32871 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:04:04,868 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:32871 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:04:04,868 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:32871 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:04:04,868 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:32871 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:04:04,868 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:32871 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:04:04,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:04,868 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:32871 2024-11-16T04:04:04,877 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[12: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=false, clientTlsEnabled=true] Thread=49 (was 48) - Thread LEAK? -, OpenFileDescriptor=348 (was 348), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6876 (was 6876) 2024-11-16T04:04:04,883 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=49, OpenFileDescriptor=348, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6876 2024-11-16T04:04:04,884 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:04,884 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38851 2024-11-16T04:04:04,885 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ed269bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:04,885 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:04,885 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:04,885 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:04,886 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:38851 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:04:04,886 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38851 2024-11-16T04:04:04,893 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=49 (was 49), OpenFileDescriptor=348 (was 348), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6876 (was 6876) 2024-11-16T04:04:04,900 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=49, OpenFileDescriptor=348, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6876 2024-11-16T04:04:04,901 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:04,901 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44363 2024-11-16T04:04:04,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5de67834, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:04,902 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:04,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:04,903 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:04,935 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:04,936 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:04,949 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:04,961 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:04,962 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x1a015613, L:/127.0.0.1:44363 - R:/127.0.0.1:43162] 2024-11-16T04:04:04,987 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:05,046 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44363 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=144ms, rpcTimeout=100ms 2024-11-16T04:04:05,157 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44363 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:05,267 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44363 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:05,376 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44363 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:05,486 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44363 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:05,596 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44363 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:05,706 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44363 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:05,816 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:33774 deadline: 1731729844915 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:33774 2024-11-16T04:04:05,816 WARN [TestNettyTlsIPC-1-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:05,817 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44363 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:05,816 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:33774 deadline: 1731729844915 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:33774 2024-11-16T04:04:05,817 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:33774 deadline: 1731729844915 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:33774 2024-11-16T04:04:05,817 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:33774 deadline: 1731729844915 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:33774 2024-11-16T04:04:05,817 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:33774 deadline: 1731729844915 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:33774 2024-11-16T04:04:05,818 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:33774 deadline: 1731729844916 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:33774 2024-11-16T04:04:05,818 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:33774 deadline: 1731729844916 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:33774 2024-11-16T04:04:05,818 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:33774 deadline: 1731729844916 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:33774 2024-11-16T04:04:05,819 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:33774 deadline: 1731729844916 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:33774 2024-11-16T04:04:05,926 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44363 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:05,988 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:43162 deadline: 1731729845147 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:43162 2024-11-16T04:04:05,988 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:43162 deadline: 1731729845258 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:43162 2024-11-16T04:04:05,988 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:04:05,988 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:43162 deadline: 1731729845368 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:43162 2024-11-16T04:04:05,989 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:43162 deadline: 1731729845477 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:43162 2024-11-16T04:04:05,989 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:43162 deadline: 1731729845588 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:43162 2024-11-16T04:04:05,989 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:43162 deadline: 1731729845698 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:43162 2024-11-16T04:04:05,989 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:43162 deadline: 1731729845809 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:43162 2024-11-16T04:04:05,989 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:43162 deadline: 1731729845919 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:43162 2024-11-16T04:04:06,036 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44363 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:06,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:06,036 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44363 2024-11-16T04:04:06,046 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=49 (was 49), OpenFileDescriptor=348 (was 348), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6875 (was 6876) 2024-11-16T04:04:06,052 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=49, OpenFileDescriptor=348, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6875 2024-11-16T04:04:06,053 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:06,053 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37869 2024-11-16T04:04:06,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46a386ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:06,053 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:06,054 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:06,054 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:06,080 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:06,080 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:06,094 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:06,106 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:06,106 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x5d5537eb, L:/127.0.0.1:37869 - R:/127.0.0.1:50006] 2024-11-16T04:04:06,132 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:06,133 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:06,133 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:50006 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:06,134 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:06,134 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:06,134 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:06,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:06,139 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50 (was 49) - Thread LEAK? -, OpenFileDescriptor=349 (was 348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6875 (was 6875) 2024-11-16T04:04:06,145 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50, OpenFileDescriptor=349, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6875 2024-11-16T04:04:06,146 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:06,147 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37553 2024-11-16T04:04:06,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dd7ed30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:06,147 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:06,148 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:06,148 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:06,177 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:06,177 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:06,190 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:06,202 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:06,202 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x76792556, L:/127.0.0.1:37553 - R:/127.0.0.1:60250] 2024-11-16T04:04:06,228 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:06,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:06,237 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37553 2024-11-16T04:04:06,243 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50 (was 50), OpenFileDescriptor=349 (was 349), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6869 (was 6875) 2024-11-16T04:04:06,248 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50, OpenFileDescriptor=349, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6869 2024-11-16T04:04:06,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27dd046f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:06,249 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:06,250 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40519 2024-11-16T04:04:06,250 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:06,250 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:06,251 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:06,275 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:06,276 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:06,289 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:06,301 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:06,301 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x33f2f86d, L:/127.0.0.1:40519 - R:/127.0.0.1:57648] 2024-11-16T04:04:06,330 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:06,331 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:06,331 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:57648 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:06,332 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:06,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:06,332 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40519 2024-11-16T04:04:06,339 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50 (was 50), OpenFileDescriptor=349 (was 349), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6864 (was 6869) 2024-11-16T04:04:06,346 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50, OpenFileDescriptor=349, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6864 2024-11-16T04:04:06,347 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:06,347 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33691 2024-11-16T04:04:06,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2745a022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:06,348 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:06,348 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:06,348 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:06,376 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:06,376 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:06,400 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:06,413 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:06,413 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xfb328358, L:/127.0.0.1:33691 - R:/127.0.0.1:34394] 2024-11-16T04:04:06,438 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:06,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:06,441 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33691 2024-11-16T04:04:06,446 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50 (was 50), OpenFileDescriptor=349 (was 349), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6864 (was 6864) 2024-11-16T04:04:06,450 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50, OpenFileDescriptor=349, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6863 2024-11-16T04:04:06,451 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:06,452 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40815 2024-11-16T04:04:06,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51db8485, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:06,452 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:06,452 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:06,452 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:06,479 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:06,480 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:06,492 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:06,506 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:06,506 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xc41a3e1c, L:/127.0.0.1:40815 - R:/127.0.0.1:35580] 2024-11-16T04:04:06,533 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:06,533 WARN [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:35580; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:06,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:06,533 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:40815 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:04:06,534 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40815 2024-11-16T04:04:06,538 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50 (was 50), OpenFileDescriptor=349 (was 349), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6863 (was 6863) 2024-11-16T04:04:06,543 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50, OpenFileDescriptor=349, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6863 2024-11-16T04:04:06,544 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:06,545 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43241 2024-11-16T04:04:06,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e19ca37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:06,545 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:06,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:06,546 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:06,574 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:06,574 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:06,587 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:06,600 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:06,600 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x2a6552d5, L:/127.0.0.1:43241 - R:/127.0.0.1:48722] 2024-11-16T04:04:06,625 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:04:06,625 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:04:06,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:06,630 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50 (was 50), OpenFileDescriptor=350 (was 349) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6863 (was 6863) 2024-11-16T04:04:06,634 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50, OpenFileDescriptor=350, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6863 2024-11-16T04:04:06,635 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:06,635 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45577 2024-11-16T04:04:06,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@773d7b4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:06,635 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:06,636 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:06,636 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:06,663 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:06,663 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:06,675 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:06,689 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:06,689 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x97d1c86b, L:/127.0.0.1:45577 - R:/127.0.0.1:49612] 2024-11-16T04:04:06,715 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:04:06,715 WARN [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:49612 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:49612 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:06,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:06,720 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50 (was 50), OpenFileDescriptor=351 (was 350) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6863 (was 6863) 2024-11-16T04:04:06,724 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50, OpenFileDescriptor=351, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6863 2024-11-16T04:04:06,725 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:06,725 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44277 2024-11-16T04:04:06,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ed7d988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:06,725 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:06,726 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:06,726 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:06,755 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:06,755 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:06,768 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:06,780 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:06,780 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xebec70c3, L:/127.0.0.1:44277 - R:/127.0.0.1:33468] 2024-11-16T04:04:06,810 WARN [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:33468 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:33468 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:06,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:06,811 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44277 2024-11-16T04:04:06,816 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50 (was 50), OpenFileDescriptor=351 (was 351), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6862 (was 6863) 2024-11-16T04:04:06,821 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50, OpenFileDescriptor=351, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6861 2024-11-16T04:04:06,822 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:06,822 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39501 2024-11-16T04:04:06,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24354329, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:06,822 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:06,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:06,823 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:06,850 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:06,851 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:06,863 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:06,876 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:06,876 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xb98078da, L:/127.0.0.1:39501 - R:/127.0.0.1:40526] 2024-11-16T04:04:06,902 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:06,989 WARN [TestNettyTlsIPC-1-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:07,004 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:07,004 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:07,005 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:07,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:07,016 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50 (was 50), OpenFileDescriptor=352 (was 351) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6861 (was 6861) 2024-11-16T04:04:07,024 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50, OpenFileDescriptor=352, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6861 2024-11-16T04:04:07,025 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:07,025 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42037 2024-11-16T04:04:07,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dbbf639, compressor=org.apache.hadoop.io.compress.GzipCodec@3907d9d9, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:07,025 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:07,025 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:07,026 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:07,052 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:07,052 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:07,065 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:07,077 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:07,077 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xc874f2fa, L:/127.0.0.1:42037 - R:/127.0.0.1:56646] 2024-11-16T04:04:07,102 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:07,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:07,104 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42037 2024-11-16T04:04:07,109 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50 (was 50), OpenFileDescriptor=352 (was 352), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6861 (was 6861) 2024-11-16T04:04:07,114 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50, OpenFileDescriptor=352, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6860 2024-11-16T04:04:07,115 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:07,115 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45359 2024-11-16T04:04:07,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@638e18f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:07,115 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:07,115 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:07,116 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:07,141 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:07,141 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:07,155 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:07,168 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:07,168 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x23be8643, L:/127.0.0.1:45359 - R:/127.0.0.1:34534] 2024-11-16T04:04:07,194 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:07,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:07,195 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45359 2024-11-16T04:04:07,200 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50 (was 50), OpenFileDescriptor=352 (was 352), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6861 (was 6860) - AvailableMemoryMB LEAK? - 2024-11-16T04:04:07,205 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50, OpenFileDescriptor=352, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6860 2024-11-16T04:04:07,205 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:07,206 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40959 2024-11-16T04:04:07,206 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52121bd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:07,206 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:07,206 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:07,207 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:07,247 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:07,247 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:07,269 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:07,281 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:07,282 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x9787f791, L:/127.0.0.1:40959 - R:/127.0.0.1:52480] 2024-11-16T04:04:07,308 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:52480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:07,308 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:07,309 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:52480 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:07,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:07,309 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:07,309 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40959 2024-11-16T04:04:07,315 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50 (was 50), OpenFileDescriptor=352 (was 352), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6860 (was 6860) 2024-11-16T04:04:07,319 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50, OpenFileDescriptor=352, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6859 2024-11-16T04:04:07,320 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:07,321 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38577 2024-11-16T04:04:07,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:07,321 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:07,321 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:07,321 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:07,346 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:07,347 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:07,360 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:07,374 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:07,374 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x5ab55ee0, L:/127.0.0.1:38577 - R:/127.0.0.1:59092] 2024-11-16T04:04:07,400 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:07,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:07,401 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38577 2024-11-16T04:04:07,406 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50 (was 50), OpenFileDescriptor=352 (was 352), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6859 (was 6859) 2024-11-16T04:04:07,411 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50, OpenFileDescriptor=352, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6859 2024-11-16T04:04:07,411 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:07,412 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39211 2024-11-16T04:04:07,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@315b60e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:07,412 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:07,412 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:07,413 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:07,437 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:07,437 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:07,451 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:07,463 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:07,463 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x0c61edff, L:/127.0.0.1:39211 - R:/127.0.0.1:60758] 2024-11-16T04:04:07,489 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:07,489 WARN [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:07,490 INFO [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-b276830de6dfa45905ab4c55c33a6ebf-9721efc611d5e315-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:04:07,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:07,490 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:07,490 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39211 2024-11-16T04:04:07,496 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50 (was 50), OpenFileDescriptor=352 (was 352), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6859 (was 6859) 2024-11-16T04:04:07,502 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=50, OpenFileDescriptor=352, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6859 2024-11-16T04:04:07,503 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:07,503 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44523 2024-11-16T04:04:07,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71245b6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:07,503 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:07,504 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:07,504 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:07,529 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:07,529 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:07,542 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:07,556 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:07,556 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x89829116, L:/127.0.0.1:44523 - R:/127.0.0.1:45064] 2024-11-16T04:04:07,582 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:45064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:07,637 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44523 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=132ms, rpcTimeout=100ms 2024-11-16T04:04:07,637 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44523 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=132ms, rpcTimeout=100ms 2024-11-16T04:04:07,637 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44523 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=132ms, rpcTimeout=100ms 2024-11-16T04:04:07,637 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44523 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=132ms, rpcTimeout=100ms 2024-11-16T04:04:07,637 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44523 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=132ms, rpcTimeout=100ms 2024-11-16T04:04:07,637 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44523 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=132ms, rpcTimeout=100ms 2024-11-16T04:04:07,637 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44523 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=132ms, rpcTimeout=100ms 2024-11-16T04:04:07,637 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44523 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=132ms, rpcTimeout=100ms 2024-11-16T04:04:07,637 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44523 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=132ms, rpcTimeout=100ms 2024-11-16T04:04:07,637 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44523 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=132ms, rpcTimeout=100ms 2024-11-16T04:04:07,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:07,637 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44523 2024-11-16T04:04:07,643 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[13: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=true] Thread=51 (was 50) - Thread LEAK? -, OpenFileDescriptor=352 (was 352), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6858 (was 6859) 2024-11-16T04:04:07,649 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=51, OpenFileDescriptor=352, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6858 2024-11-16T04:04:07,650 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:07,651 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46167 2024-11-16T04:04:07,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6691e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:07,651 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:07,651 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:07,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:07,652 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:46167 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:04:07,652 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46167 2024-11-16T04:04:07,659 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=51 (was 51), OpenFileDescriptor=352 (was 352), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6858 (was 6858) 2024-11-16T04:04:07,665 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=51, OpenFileDescriptor=352, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6858 2024-11-16T04:04:07,666 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:07,667 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45981 2024-11-16T04:04:07,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d049948, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:07,667 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:07,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:07,686 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:07,698 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:07,699 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x8828ef19, L:/127.0.0.1:45981 - R:/127.0.0.1:56070] 2024-11-16T04:04:07,699 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:07,776 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45981 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:04:07,886 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45981 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:07,996 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45981 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:08,106 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45981 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:08,216 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45981 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:08,327 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45981 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:08,436 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45981 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:08,546 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45981 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:08,584 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:45064 deadline: 1731729847683 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:45064 2024-11-16T04:04:08,584 WARN [TestNettyTlsIPC-1-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:08,584 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:45064 deadline: 1731729847683 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:45064 2024-11-16T04:04:08,584 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:45064 deadline: 1731729847683 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:45064 2024-11-16T04:04:08,584 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:45064 deadline: 1731729847683 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:45064 2024-11-16T04:04:08,585 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:45064 deadline: 1731729847683 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:45064 2024-11-16T04:04:08,585 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:45064 deadline: 1731729847684 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:45064 2024-11-16T04:04:08,585 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:45064 deadline: 1731729847684 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:45064 2024-11-16T04:04:08,585 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:45064 deadline: 1731729847684 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:45064 2024-11-16T04:04:08,585 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:45064 deadline: 1731729847684 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:45064 2024-11-16T04:04:08,656 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45981 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:08,700 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:56070 deadline: 1731729847877 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:56070 2024-11-16T04:04:08,700 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:04:08,700 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:56070 deadline: 1731729847987 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:56070 2024-11-16T04:04:08,700 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:56070 deadline: 1731729848096 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:56070 2024-11-16T04:04:08,700 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:56070 deadline: 1731729848207 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:56070 2024-11-16T04:04:08,701 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:56070 deadline: 1731729848317 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:56070 2024-11-16T04:04:08,701 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:56070 deadline: 1731729848427 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:56070 2024-11-16T04:04:08,701 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:56070 deadline: 1731729848538 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:56070 2024-11-16T04:04:08,701 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:56070 deadline: 1731729848648 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:56070 2024-11-16T04:04:08,766 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45981 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:08,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:08,767 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45981 2024-11-16T04:04:08,779 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=51 (was 51), OpenFileDescriptor=352 (was 352), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6857 (was 6858) 2024-11-16T04:04:08,783 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=51, OpenFileDescriptor=352, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6857 2024-11-16T04:04:08,784 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:08,785 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39467 2024-11-16T04:04:08,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2275a437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:08,785 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:08,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:08,800 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:08,813 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:08,813 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xced03be5, L:/127.0.0.1:39467 - R:/127.0.0.1:58052] 2024-11-16T04:04:08,813 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:08,814 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:08,814 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:58052 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:08,815 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:08,815 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:08,815 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:08,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:08,821 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=52 (was 51) - Thread LEAK? -, OpenFileDescriptor=353 (was 352) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6857 (was 6857) 2024-11-16T04:04:08,825 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=52, OpenFileDescriptor=353, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6857 2024-11-16T04:04:08,826 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:08,826 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37809 2024-11-16T04:04:08,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20ee634d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:08,827 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:08,827 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:08,844 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:08,856 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:08,857 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xefd749f9, L:/127.0.0.1:37809 - R:/127.0.0.1:42936] 2024-11-16T04:04:08,857 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:08,863 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:08,864 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37809 2024-11-16T04:04:08,869 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=52 (was 52), OpenFileDescriptor=353 (was 353), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6857 (was 6857) 2024-11-16T04:04:08,873 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=52, OpenFileDescriptor=353, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6857 2024-11-16T04:04:08,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@527feadf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:08,874 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:08,875 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40085 2024-11-16T04:04:08,875 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:08,875 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:08,891 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:08,903 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:08,903 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xb8dd5c9e, L:/127.0.0.1:40085 - R:/127.0.0.1:58388] 2024-11-16T04:04:08,903 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:08,904 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:08,904 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:58388 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:08,905 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:08,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:08,905 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40085 2024-11-16T04:04:08,910 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=52 (was 52), OpenFileDescriptor=353 (was 353), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6857 (was 6857) 2024-11-16T04:04:08,914 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=52, OpenFileDescriptor=353, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6857 2024-11-16T04:04:08,915 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:08,915 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46801 2024-11-16T04:04:08,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ad2d45c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:08,916 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:08,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:08,932 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:08,944 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:08,944 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xf0dd33e8, L:/127.0.0.1:46801 - R:/127.0.0.1:46780] 2024-11-16T04:04:08,945 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:08,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:08,947 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46801 2024-11-16T04:04:08,952 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=52 (was 52), OpenFileDescriptor=353 (was 353), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6857 (was 6857) 2024-11-16T04:04:08,956 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=52, OpenFileDescriptor=353, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6857 2024-11-16T04:04:08,957 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:08,957 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42775 2024-11-16T04:04:08,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f636fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:08,958 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:08,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:08,975 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:08,987 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:08,987 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x6bacbd4d, L:/127.0.0.1:42775 - R:/127.0.0.1:45392] 2024-11-16T04:04:08,987 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:45392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:08,987 WARN [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:45392; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:08,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:08,988 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:42775 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:04:08,988 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42775 2024-11-16T04:04:08,993 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=52 (was 52), OpenFileDescriptor=353 (was 353), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6857 (was 6857) 2024-11-16T04:04:08,997 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=52, OpenFileDescriptor=353, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6857 2024-11-16T04:04:08,998 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:08,998 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45477 2024-11-16T04:04:08,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78ba7114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:08,998 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:08,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:09,015 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:09,027 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:09,028 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x10510cca, L:/127.0.0.1:45477 - R:/127.0.0.1:50682] 2024-11-16T04:04:09,028 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:04:09,028 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:04:09,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:09,033 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=52 (was 52), OpenFileDescriptor=354 (was 353) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6856 (was 6857) 2024-11-16T04:04:09,037 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=52, OpenFileDescriptor=354, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6856 2024-11-16T04:04:09,038 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:09,038 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39407 2024-11-16T04:04:09,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ad91733, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:09,039 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:09,039 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:09,055 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:09,068 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:09,068 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x9ac282f9, L:/127.0.0.1:39407 - R:/127.0.0.1:45878] 2024-11-16T04:04:09,068 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:04:09,068 WARN [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:45878 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:45878 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:09,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:09,074 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=52 (was 52), OpenFileDescriptor=355 (was 354) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6856 (was 6856) 2024-11-16T04:04:09,078 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=52, OpenFileDescriptor=355, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6856 2024-11-16T04:04:09,079 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:09,079 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34411 2024-11-16T04:04:09,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e2452b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:09,079 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:09,079 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:09,096 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:09,108 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:09,109 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xe4cf900c, L:/127.0.0.1:34411 - R:/127.0.0.1:41184] 2024-11-16T04:04:09,109 WARN [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:41184 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:41184 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:09,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:09,110 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34411 2024-11-16T04:04:09,115 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=52 (was 52), OpenFileDescriptor=355 (was 355), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6856 (was 6856) 2024-11-16T04:04:09,120 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=52, OpenFileDescriptor=355, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6856 2024-11-16T04:04:09,121 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:09,122 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37413 2024-11-16T04:04:09,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76844807, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:09,122 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:09,123 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:09,136 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:09,148 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:09,148 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x9fc7926d, L:/127.0.0.1:37413 - R:/127.0.0.1:54264] 2024-11-16T04:04:09,148 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:09,250 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:09,250 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:09,250 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:09,250 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:09,257 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=53 (was 52) - Thread LEAK? -, OpenFileDescriptor=356 (was 355) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 127) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6856 (was 6856) 2024-11-16T04:04:09,264 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=53, OpenFileDescriptor=356, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6856 2024-11-16T04:04:09,265 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:09,265 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44469 2024-11-16T04:04:09,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1600fc89, compressor=org.apache.hadoop.io.compress.GzipCodec@c273b12, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:09,266 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:09,266 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:09,286 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:09,298 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:09,298 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x89c5f579, L:/127.0.0.1:44469 - R:/127.0.0.1:57852] 2024-11-16T04:04:09,298 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:09,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:09,300 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44469 2024-11-16T04:04:09,305 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=53 (was 53), OpenFileDescriptor=356 (was 356), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6856 (was 6856) 2024-11-16T04:04:09,310 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=53, OpenFileDescriptor=356, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6856 2024-11-16T04:04:09,311 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:09,311 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43043 2024-11-16T04:04:09,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79e2dd9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:09,311 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:09,312 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:09,329 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:09,341 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:09,341 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x55f5c0ee, L:/127.0.0.1:43043 - R:/127.0.0.1:45792] 2024-11-16T04:04:09,342 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:45792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:09,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:09,343 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43043 2024-11-16T04:04:09,347 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=53 (was 53), OpenFileDescriptor=356 (was 356), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6856 (was 6856) 2024-11-16T04:04:09,352 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=53, OpenFileDescriptor=356, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6856 2024-11-16T04:04:09,352 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:09,353 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41571 2024-11-16T04:04:09,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66349439, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:09,353 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:09,353 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:09,369 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:09,382 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:09,382 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x9564a637, L:/127.0.0.1:41571 - R:/127.0.0.1:55010] 2024-11-16T04:04:09,382 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:09,383 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:09,383 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:55010 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:09,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:09,383 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:09,383 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41571 2024-11-16T04:04:09,388 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=53 (was 53), OpenFileDescriptor=356 (was 356), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6856 (was 6856) 2024-11-16T04:04:09,393 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=53, OpenFileDescriptor=356, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6856 2024-11-16T04:04:09,394 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:09,394 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35523 2024-11-16T04:04:09,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:09,395 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:09,395 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:09,410 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:09,423 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:09,423 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xab8cea41, L:/127.0.0.1:35523 - R:/127.0.0.1:35308] 2024-11-16T04:04:09,423 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:09,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:09,424 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35523 2024-11-16T04:04:09,429 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=53 (was 53), OpenFileDescriptor=356 (was 356), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6856 (was 6856) 2024-11-16T04:04:09,433 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=53, OpenFileDescriptor=356, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6856 2024-11-16T04:04:09,434 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:09,435 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45329 2024-11-16T04:04:09,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a14a613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:09,435 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:09,435 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:09,452 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:09,464 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:09,464 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xc7147af0, L:/127.0.0.1:45329 - R:/127.0.0.1:33526] 2024-11-16T04:04:09,464 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:09,464 WARN [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:09,465 INFO [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-0f69c5163d75a9eb89b07dec179cc9d5-29ac1480e370e005-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:04:09,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:09,465 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:09,465 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45329 2024-11-16T04:04:09,470 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=53 (was 53), OpenFileDescriptor=356 (was 356), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6856 (was 6856) 2024-11-16T04:04:09,474 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=53, OpenFileDescriptor=356, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6856 2024-11-16T04:04:09,476 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:09,476 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45605 2024-11-16T04:04:09,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@539301b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:09,476 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:09,477 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:09,492 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:09,505 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:09,505 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x9fca6888, L:/127.0.0.1:45605 - R:/127.0.0.1:52626] 2024-11-16T04:04:09,505 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:52626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:09,587 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45605 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:09,588 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45605 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:09,588 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45605 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:09,588 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45605 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:09,588 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45605 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:09,588 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45605 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:09,588 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45605 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:09,588 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45605 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:09,588 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45605 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:09,588 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45605 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:09,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:09,588 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45605 2024-11-16T04:04:09,599 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[14: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@3a320ade, acceptPlainText=true, clientTlsEnabled=false] Thread=54 (was 53) - Thread LEAK? -, OpenFileDescriptor=356 (was 356), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6856 (was 6856) 2024-11-16T04:04:09,604 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54, OpenFileDescriptor=356, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6856 2024-11-16T04:04:09,702 WARN [TestNettyTlsIPC-1-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:09,754 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:09,755 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40635 2024-11-16T04:04:09,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ed2de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:09,755 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:09,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:09,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:09,756 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:40635 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:04:09,756 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40635 2024-11-16T04:04:09,761 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=53 (was 54), OpenFileDescriptor=356 (was 356), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6860 (was 6856) - AvailableMemoryMB LEAK? - 2024-11-16T04:04:09,766 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=53, OpenFileDescriptor=356, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6860 2024-11-16T04:04:09,767 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:09,768 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43699 2024-11-16T04:04:09,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a105dce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:09,768 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:09,768 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:09,768 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:09,795 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:09,796 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:09,809 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:09,821 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:09,822 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x94f89ff5, L:/127.0.0.1:43699 - R:/127.0.0.1:51752] 2024-11-16T04:04:09,849 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:09,906 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:43699 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=138ms, rpcTimeout=100ms 2024-11-16T04:04:10,016 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:43699 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:10,127 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:43699 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:10,237 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:43699 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:10,346 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:43699 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:10,456 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:43699 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:10,506 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52626 deadline: 1731729849606 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52626 2024-11-16T04:04:10,506 WARN [TestNettyTlsIPC-1-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:10,506 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52626 deadline: 1731729849606 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52626 2024-11-16T04:04:10,506 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52626 deadline: 1731729849606 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52626 2024-11-16T04:04:10,506 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52626 deadline: 1731729849606 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52626 2024-11-16T04:04:10,507 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52626 deadline: 1731729849606 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52626 2024-11-16T04:04:10,507 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52626 deadline: 1731729849606 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52626 2024-11-16T04:04:10,507 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52626 deadline: 1731729849606 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52626 2024-11-16T04:04:10,507 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52626 deadline: 1731729849606 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52626 2024-11-16T04:04:10,507 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52626 deadline: 1731729849607 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52626 2024-11-16T04:04:10,566 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:43699 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:10,676 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:43699 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:10,786 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:43699 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:10,851 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51752 deadline: 1731729850007 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51752 2024-11-16T04:04:10,851 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51752 deadline: 1731729850117 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51752 2024-11-16T04:04:10,851 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:04:10,851 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51752 deadline: 1731729850228 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51752 2024-11-16T04:04:10,852 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51752 deadline: 1731729850338 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51752 2024-11-16T04:04:10,852 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51752 deadline: 1731729850447 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51752 2024-11-16T04:04:10,852 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51752 deadline: 1731729850557 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51752 2024-11-16T04:04:10,853 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51752 deadline: 1731729850667 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51752 2024-11-16T04:04:10,853 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51752 deadline: 1731729850777 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51752 2024-11-16T04:04:10,896 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:43699 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:10,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:10,896 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43699 2024-11-16T04:04:10,910 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=53 (was 53), OpenFileDescriptor=356 (was 356), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6856 (was 6860) 2024-11-16T04:04:10,917 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=53, OpenFileDescriptor=356, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6856 2024-11-16T04:04:10,918 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:10,919 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39027 2024-11-16T04:04:10,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e9bfbef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:10,919 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:10,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:10,920 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:10,948 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:10,949 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:10,961 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:10,973 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:10,974 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x55835773, L:/127.0.0.1:39027 - R:/127.0.0.1:59014] 2024-11-16T04:04:11,006 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:11,007 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:11,008 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:59014 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:11,008 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:11,009 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:11,009 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:11,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:11,014 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54 (was 53) - Thread LEAK? -, OpenFileDescriptor=357 (was 356) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6856 (was 6856) 2024-11-16T04:04:11,018 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54, OpenFileDescriptor=357, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6856 2024-11-16T04:04:11,020 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:11,020 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44903 2024-11-16T04:04:11,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b067e7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:11,020 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:11,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:11,021 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:11,046 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:11,046 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:11,058 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:11,070 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:11,070 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x21831a5f, L:/127.0.0.1:44903 - R:/127.0.0.1:56586] 2024-11-16T04:04:11,095 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:11,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:11,106 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44903 2024-11-16T04:04:11,111 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54 (was 54), OpenFileDescriptor=357 (was 357), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6856 (was 6856) 2024-11-16T04:04:11,116 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54, OpenFileDescriptor=357, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6855 2024-11-16T04:04:11,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1185ca88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:11,118 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:11,118 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41819 2024-11-16T04:04:11,118 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:11,119 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:11,119 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:11,148 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:11,149 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:11,161 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:11,174 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:11,174 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x4cb34aa1, L:/127.0.0.1:41819 - R:/127.0.0.1:40014] 2024-11-16T04:04:11,200 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:11,201 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:11,201 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:40014 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:11,201 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:11,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:11,201 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41819 2024-11-16T04:04:11,206 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54 (was 54), OpenFileDescriptor=357 (was 357), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6855 (was 6855) 2024-11-16T04:04:11,211 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54, OpenFileDescriptor=357, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6855 2024-11-16T04:04:11,212 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:11,213 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37793 2024-11-16T04:04:11,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b5b5e8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:11,213 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:11,213 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:11,213 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:11,242 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:11,242 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:11,255 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:11,267 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:11,267 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x037f4907, L:/127.0.0.1:37793 - R:/127.0.0.1:59710] 2024-11-16T04:04:11,292 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:11,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:11,295 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37793 2024-11-16T04:04:11,300 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54 (was 54), OpenFileDescriptor=357 (was 357), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6855 (was 6855) 2024-11-16T04:04:11,305 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54, OpenFileDescriptor=357, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6855 2024-11-16T04:04:11,306 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:11,306 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46541 2024-11-16T04:04:11,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9d4d859, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:11,307 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:11,307 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:11,307 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:11,336 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:11,336 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:11,350 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:11,362 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:11,362 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x1f457b27, L:/127.0.0.1:46541 - R:/127.0.0.1:56640] 2024-11-16T04:04:11,392 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:11,392 WARN [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:56640; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:11,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:11,393 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:46541 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:04:11,393 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46541 2024-11-16T04:04:11,398 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54 (was 54), OpenFileDescriptor=357 (was 357), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6855 (was 6855) 2024-11-16T04:04:11,402 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54, OpenFileDescriptor=357, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6855 2024-11-16T04:04:11,403 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:11,403 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41671 2024-11-16T04:04:11,404 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c1a5f4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:11,404 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:11,404 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:11,404 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:11,438 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:11,438 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:11,451 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:11,464 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:11,464 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x961de9e6, L:/127.0.0.1:41671 - R:/127.0.0.1:47214] 2024-11-16T04:04:11,490 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:04:11,491 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:04:11,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:11,496 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54 (was 54), OpenFileDescriptor=358 (was 357) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6855 (was 6855) 2024-11-16T04:04:11,500 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54, OpenFileDescriptor=358, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6855 2024-11-16T04:04:11,501 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:11,501 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39067 2024-11-16T04:04:11,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40ac4720, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:11,501 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:11,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:11,502 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:11,531 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:11,531 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:11,546 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:11,558 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:11,559 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x3723af6a, L:/127.0.0.1:39067 - R:/127.0.0.1:50386] 2024-11-16T04:04:11,584 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:04:11,584 WARN [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:50386 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:50386 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:11,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:11,591 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54 (was 54), OpenFileDescriptor=359 (was 358) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6855 (was 6855) 2024-11-16T04:04:11,595 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54, OpenFileDescriptor=359, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6855 2024-11-16T04:04:11,597 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:11,597 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41853 2024-11-16T04:04:11,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28e4555c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:11,597 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:11,597 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:11,598 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:11,627 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:11,627 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:11,640 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:11,653 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:11,653 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x48e94554, L:/127.0.0.1:41853 - R:/127.0.0.1:51926] 2024-11-16T04:04:11,678 WARN [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:51926 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:51926 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:11,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:11,679 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41853 2024-11-16T04:04:11,684 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54 (was 54), OpenFileDescriptor=359 (was 359), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6855 (was 6855) 2024-11-16T04:04:11,689 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54, OpenFileDescriptor=359, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6855 2024-11-16T04:04:11,690 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:11,690 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39367 2024-11-16T04:04:11,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d4534c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:11,690 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:11,691 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:11,691 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:11,720 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:11,721 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:11,733 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:11,745 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:11,746 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x8655de73, L:/127.0.0.1:39367 - R:/127.0.0.1:54726] 2024-11-16T04:04:11,771 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:11,854 WARN [TestNettyTlsIPC-1-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:11,873 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:11,874 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:11,874 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:11,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:11,886 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54 (was 54), OpenFileDescriptor=360 (was 359) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6855 (was 6855) 2024-11-16T04:04:11,892 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54, OpenFileDescriptor=360, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6855 2024-11-16T04:04:11,893 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:11,894 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38435 2024-11-16T04:04:11,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eb346f1, compressor=org.apache.hadoop.io.compress.GzipCodec@2e71bdac, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:11,894 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:11,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:11,894 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:11,921 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:11,921 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:11,935 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:11,947 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:11,947 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x6e60836f, L:/127.0.0.1:38435 - R:/127.0.0.1:41832] 2024-11-16T04:04:11,980 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:11,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:11,981 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38435 2024-11-16T04:04:11,986 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54 (was 54), OpenFileDescriptor=360 (was 360), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6855 (was 6855) 2024-11-16T04:04:11,991 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54, OpenFileDescriptor=360, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6855 2024-11-16T04:04:11,993 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:11,993 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36093 2024-11-16T04:04:11,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@181626ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:11,993 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:11,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:11,994 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:12,023 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:12,023 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:12,036 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:12,049 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:12,050 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xf0fc5645, L:/127.0.0.1:36093 - R:/127.0.0.1:40406] 2024-11-16T04:04:12,076 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:12,077 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:12,077 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36093 2024-11-16T04:04:12,082 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54 (was 54), OpenFileDescriptor=360 (was 360), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6855 (was 6855) 2024-11-16T04:04:12,086 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54, OpenFileDescriptor=360, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6855 2024-11-16T04:04:12,088 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:12,088 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35461 2024-11-16T04:04:12,088 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ca024b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:12,088 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:12,089 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:12,089 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:12,119 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:12,119 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:12,132 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:12,145 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:12,145 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x81601c7a, L:/127.0.0.1:35461 - R:/127.0.0.1:52266] 2024-11-16T04:04:12,175 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:52266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:12,176 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:12,176 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:52266 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:12,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:12,177 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:12,177 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35461 2024-11-16T04:04:12,182 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54 (was 54), OpenFileDescriptor=360 (was 360), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6855 (was 6855) 2024-11-16T04:04:12,187 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54, OpenFileDescriptor=360, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6855 2024-11-16T04:04:12,188 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:12,188 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42479 2024-11-16T04:04:12,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:12,188 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:12,188 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:12,189 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:12,215 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:12,215 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:12,229 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:12,241 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:12,242 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x84c5660a, L:/127.0.0.1:42479 - R:/127.0.0.1:55528] 2024-11-16T04:04:12,269 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:12,270 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:12,270 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42479 2024-11-16T04:04:12,276 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54 (was 54), OpenFileDescriptor=360 (was 360), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6855 (was 6855) 2024-11-16T04:04:12,280 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54, OpenFileDescriptor=360, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6855 2024-11-16T04:04:12,281 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:12,282 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38721 2024-11-16T04:04:12,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10850050, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:12,282 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:12,282 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:12,283 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:12,307 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:12,308 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:12,321 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:12,333 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:12,334 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x7a51adae, L:/127.0.0.1:38721 - R:/127.0.0.1:37778] 2024-11-16T04:04:12,364 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:12,364 WARN [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:12,368 INFO [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-4139e8ffb57dc9387f121c71d2a8316f-a7d5d559b9ad8560-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:04:12,368 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:12,368 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:12,368 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38721 2024-11-16T04:04:12,373 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54 (was 54), OpenFileDescriptor=360 (was 360), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6854 (was 6855) 2024-11-16T04:04:12,378 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=54, OpenFileDescriptor=360, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6854 2024-11-16T04:04:12,379 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:12,380 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34691 2024-11-16T04:04:12,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@323f594c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:12,380 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:12,380 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:12,380 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:12,408 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:12,408 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:12,423 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:12,435 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:12,436 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x4d3f75d1, L:/127.0.0.1:34691 - R:/127.0.0.1:48598] 2024-11-16T04:04:12,463 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:12,516 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34691 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:04:12,516 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34691 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=136ms, rpcTimeout=100ms 2024-11-16T04:04:12,516 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34691 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=136ms, rpcTimeout=100ms 2024-11-16T04:04:12,516 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34691 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:04:12,516 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34691 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:04:12,516 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34691 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:04:12,516 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34691 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:04:12,516 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34691 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:04:12,516 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34691 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:04:12,516 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34691 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:04:12,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:12,517 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34691 2024-11-16T04:04:12,523 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[15: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=false, clientTlsEnabled=true] Thread=55 (was 54) - Thread LEAK? -, OpenFileDescriptor=360 (was 360), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6853 (was 6854) 2024-11-16T04:04:12,530 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=55, OpenFileDescriptor=360, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6853 2024-11-16T04:04:12,532 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:12,532 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39645 2024-11-16T04:04:12,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69127b2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:12,533 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:12,533 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:12,534 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:12,534 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:39645 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:04:12,534 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39645 2024-11-16T04:04:12,541 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=55 (was 55), OpenFileDescriptor=360 (was 360), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6852 (was 6853) 2024-11-16T04:04:12,547 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=55, OpenFileDescriptor=360, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6852 2024-11-16T04:04:12,548 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:12,549 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42951 2024-11-16T04:04:12,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fcbdffe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:12,549 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:12,550 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:12,550 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:12,579 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:12,580 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:12,592 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:12,605 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:12,605 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x5f91ed8e, L:/127.0.0.1:42951 - R:/127.0.0.1:36704] 2024-11-16T04:04:12,631 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:12,686 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:42951 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=136ms, rpcTimeout=100ms 2024-11-16T04:04:12,796 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:42951 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:12,906 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:42951 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:13,017 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:42951 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:13,126 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:42951 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:13,236 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:42951 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:13,347 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:42951 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:13,457 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:42951 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:13,464 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:48598 deadline: 1731729852564 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:48598 2024-11-16T04:04:13,464 WARN [TestNettyTlsIPC-1-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:13,465 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:48598 deadline: 1731729852564 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:48598 2024-11-16T04:04:13,465 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:48598 deadline: 1731729852564 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:48598 2024-11-16T04:04:13,465 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:48598 deadline: 1731729852564 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:48598 2024-11-16T04:04:13,465 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:48598 deadline: 1731729852564 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:48598 2024-11-16T04:04:13,466 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:48598 deadline: 1731729852564 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:48598 2024-11-16T04:04:13,466 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:48598 deadline: 1731729852564 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:48598 2024-11-16T04:04:13,466 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:48598 deadline: 1731729852564 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:48598 2024-11-16T04:04:13,466 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:48598 deadline: 1731729852565 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:48598 2024-11-16T04:04:13,566 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:42951 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:13,632 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:36704 deadline: 1731729852787 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:36704 2024-11-16T04:04:13,633 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:36704 deadline: 1731729852897 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:36704 2024-11-16T04:04:13,633 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:36704 deadline: 1731729853007 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:36704 2024-11-16T04:04:13,633 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:04:13,633 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:36704 deadline: 1731729853117 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:36704 2024-11-16T04:04:13,633 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:36704 deadline: 1731729853227 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:36704 2024-11-16T04:04:13,633 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:36704 deadline: 1731729853338 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:36704 2024-11-16T04:04:13,633 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:36704 deadline: 1731729853448 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:36704 2024-11-16T04:04:13,633 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:36704 deadline: 1731729853558 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:36704 2024-11-16T04:04:13,676 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:42951 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:13,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:13,676 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42951 2024-11-16T04:04:13,682 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=55 (was 55), OpenFileDescriptor=360 (was 360), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6849 (was 6852) 2024-11-16T04:04:13,687 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=55, OpenFileDescriptor=360, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6849 2024-11-16T04:04:13,688 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:13,688 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40623 2024-11-16T04:04:13,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14c8ce75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:13,688 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:13,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:13,689 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:13,713 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:13,714 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:13,726 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:13,739 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:13,739 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xee5d0340, L:/127.0.0.1:40623 - R:/127.0.0.1:37500] 2024-11-16T04:04:13,765 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37500, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:13,765 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:13,766 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:37500 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:13,766 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:13,767 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:13,767 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:13,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:13,772 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56 (was 55) - Thread LEAK? -, OpenFileDescriptor=361 (was 360) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6849 (was 6849) 2024-11-16T04:04:13,778 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56, OpenFileDescriptor=361, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6849 2024-11-16T04:04:13,779 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:13,779 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33209 2024-11-16T04:04:13,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b87f1b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:13,779 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:13,780 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:13,780 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:13,805 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:13,805 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:13,818 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:13,830 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:13,830 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xf0cc1579, L:/127.0.0.1:33209 - R:/127.0.0.1:54342] 2024-11-16T04:04:13,855 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:13,863 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:13,863 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33209 2024-11-16T04:04:13,869 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56 (was 56), OpenFileDescriptor=361 (was 361), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6849 (was 6849) 2024-11-16T04:04:13,873 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56, OpenFileDescriptor=361, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6849 2024-11-16T04:04:13,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eea5cfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:13,874 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:13,875 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38561 2024-11-16T04:04:13,875 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:13,875 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:13,875 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:13,904 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:13,904 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:13,917 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:13,930 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:13,930 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xca28b6a2, L:/127.0.0.1:38561 - R:/127.0.0.1:60964] 2024-11-16T04:04:13,955 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:13,956 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:13,956 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:60964 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:13,956 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:13,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:13,956 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38561 2024-11-16T04:04:13,961 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56 (was 56), OpenFileDescriptor=361 (was 361), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6848 (was 6849) 2024-11-16T04:04:13,966 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56, OpenFileDescriptor=361, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6848 2024-11-16T04:04:13,967 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:13,967 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:32859 2024-11-16T04:04:13,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5151290f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:13,968 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:13,968 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:13,968 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:13,999 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:13,999 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:14,011 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:14,024 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:14,024 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xae06b1f3, L:/127.0.0.1:32859 - R:/127.0.0.1:36958] 2024-11-16T04:04:14,050 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:14,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:14,053 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:32859 2024-11-16T04:04:14,058 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56 (was 56), OpenFileDescriptor=361 (was 361), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6848 (was 6848) 2024-11-16T04:04:14,064 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56, OpenFileDescriptor=361, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6848 2024-11-16T04:04:14,064 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:14,065 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41269 2024-11-16T04:04:14,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69e52357, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:14,065 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:14,065 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:14,065 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:14,093 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:14,094 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:14,107 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:14,120 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:14,120 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x8ae08255, L:/127.0.0.1:41269 - R:/127.0.0.1:37900] 2024-11-16T04:04:14,145 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:14,145 WARN [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:37900; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:14,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:14,146 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:41269 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:04:14,146 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41269 2024-11-16T04:04:14,151 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56 (was 56), OpenFileDescriptor=361 (was 361), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6848 (was 6848) 2024-11-16T04:04:14,155 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56, OpenFileDescriptor=361, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=6848 2024-11-16T04:04:14,157 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:14,157 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34907 2024-11-16T04:04:14,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@305f2d33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:14,157 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:14,157 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:14,158 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:14,186 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:14,186 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:14,199 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:14,211 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:14,211 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xd996f25f, L:/127.0.0.1:34907 - R:/127.0.0.1:57296] 2024-11-16T04:04:14,236 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:04:14,236 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:04:14,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:14,241 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56 (was 56), OpenFileDescriptor=362 (was 361) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=6848 (was 6848) 2024-11-16T04:04:14,246 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56, OpenFileDescriptor=362, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6848 2024-11-16T04:04:14,247 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:14,247 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40121 2024-11-16T04:04:14,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@369d9dc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:14,247 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:14,247 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:14,248 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:14,275 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:14,275 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:14,287 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:14,299 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:14,299 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x3840077d, L:/127.0.0.1:40121 - R:/127.0.0.1:39766] 2024-11-16T04:04:14,324 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:04:14,324 WARN [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:39766 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:39766 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:14,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:14,329 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56 (was 56), OpenFileDescriptor=363 (was 362) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6848 (was 6848) 2024-11-16T04:04:14,334 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56, OpenFileDescriptor=363, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6848 2024-11-16T04:04:14,335 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:14,335 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44419 2024-11-16T04:04:14,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fb80169, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:14,335 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:14,335 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:14,336 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:14,364 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:14,364 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:14,377 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:14,391 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:14,391 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xe80744fa, L:/127.0.0.1:44419 - R:/127.0.0.1:40996] 2024-11-16T04:04:14,416 WARN [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:40996 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:40996 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:14,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:14,417 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44419 2024-11-16T04:04:14,421 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56 (was 56), OpenFileDescriptor=363 (was 363), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6848 (was 6848) 2024-11-16T04:04:14,426 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56, OpenFileDescriptor=363, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6848 2024-11-16T04:04:14,427 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:14,427 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41101 2024-11-16T04:04:14,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@766df24a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:14,427 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:14,428 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:14,428 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:14,455 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:14,455 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:14,468 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:14,480 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:14,480 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x88086ee6, L:/127.0.0.1:41101 - R:/127.0.0.1:34778] 2024-11-16T04:04:14,506 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:14,608 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:14,608 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:14,608 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:14,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:14,618 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=57 (was 56) - Thread LEAK? -, OpenFileDescriptor=364 (was 363) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6848 (was 6848) 2024-11-16T04:04:14,625 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=57, OpenFileDescriptor=364, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6847 2024-11-16T04:04:14,627 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:14,627 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33003 2024-11-16T04:04:14,627 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d38204, compressor=org.apache.hadoop.io.compress.GzipCodec@31c1515b, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:14,627 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:14,627 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:14,628 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:14,634 WARN [TestNettyTlsIPC-1-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:14,657 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:14,658 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:14,671 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:14,683 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:14,683 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xd2c432a9, L:/127.0.0.1:33003 - R:/127.0.0.1:46668] 2024-11-16T04:04:14,709 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:14,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:14,717 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33003 2024-11-16T04:04:14,721 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56 (was 57), OpenFileDescriptor=364 (was 364), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6847 (was 6847) 2024-11-16T04:04:14,726 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56, OpenFileDescriptor=364, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6847 2024-11-16T04:04:14,727 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:14,727 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33137 2024-11-16T04:04:14,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44c0883b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:14,727 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:14,728 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:14,728 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:14,756 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:14,756 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:14,768 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:14,780 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:14,780 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xa8eb643b, L:/127.0.0.1:33137 - R:/127.0.0.1:50446] 2024-11-16T04:04:14,806 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:14,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:14,807 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33137 2024-11-16T04:04:14,812 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56 (was 56), OpenFileDescriptor=364 (was 364), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6847 (was 6847) 2024-11-16T04:04:14,817 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56, OpenFileDescriptor=364, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6847 2024-11-16T04:04:14,817 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:14,818 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45919 2024-11-16T04:04:14,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64083175, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:14,818 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:14,818 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:14,818 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:14,845 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:14,846 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:14,860 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:14,872 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:14,872 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x59d17234, L:/127.0.0.1:45919 - R:/127.0.0.1:60906] 2024-11-16T04:04:14,899 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:14,900 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:14,900 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:60906 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:14,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:14,900 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:14,900 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45919 2024-11-16T04:04:14,906 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56 (was 56), OpenFileDescriptor=364 (was 364), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6847 (was 6847) 2024-11-16T04:04:14,910 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56, OpenFileDescriptor=364, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6847 2024-11-16T04:04:14,911 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:14,911 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39247 2024-11-16T04:04:14,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:14,912 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:14,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:14,912 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:14,940 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:14,940 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:14,954 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:14,966 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:14,966 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xaed65769, L:/127.0.0.1:39247 - R:/127.0.0.1:47910] 2024-11-16T04:04:14,992 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:14,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:14,993 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39247 2024-11-16T04:04:14,998 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56 (was 56), OpenFileDescriptor=364 (was 364), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6847 (was 6847) 2024-11-16T04:04:15,002 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56, OpenFileDescriptor=364, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6847 2024-11-16T04:04:15,003 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:15,003 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36653 2024-11-16T04:04:15,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe3f024, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:15,003 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:15,004 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:15,004 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:15,032 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:15,032 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:15,045 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:15,058 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:15,058 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x7f191dce, L:/127.0.0.1:36653 - R:/127.0.0.1:49136] 2024-11-16T04:04:15,085 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:15,085 WARN [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:15,085 INFO [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-b5ae00c5f883a3a33a499973aae20111-1bbfe219b73795f5-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:04:15,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:15,086 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:15,086 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36653 2024-11-16T04:04:15,091 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56 (was 56), OpenFileDescriptor=364 (was 364), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6847 (was 6847) 2024-11-16T04:04:15,096 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=56, OpenFileDescriptor=364, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6847 2024-11-16T04:04:15,097 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:15,097 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33617 2024-11-16T04:04:15,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64ba7c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:15,097 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:15,098 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:15,098 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:15,123 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:15,123 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:15,136 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:15,148 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:15,148 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xd345b405, L:/127.0.0.1:33617 - R:/127.0.0.1:42652] 2024-11-16T04:04:15,175 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:15,228 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33617 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=128ms, rpcTimeout=100ms 2024-11-16T04:04:15,228 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33617 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=128ms, rpcTimeout=100ms 2024-11-16T04:04:15,228 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33617 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:04:15,228 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33617 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:04:15,228 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33617 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:04:15,228 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33617 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:04:15,228 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33617 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:04:15,228 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33617 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:04:15,228 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33617 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:04:15,228 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33617 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:04:15,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:15,228 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33617 2024-11-16T04:04:15,238 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[16: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=true] Thread=57 (was 56) - Thread LEAK? -, OpenFileDescriptor=364 (was 364), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6847 (was 6847) 2024-11-16T04:04:15,243 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=57, OpenFileDescriptor=364, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6847 2024-11-16T04:04:15,244 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:15,244 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38405 2024-11-16T04:04:15,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@489c94b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:15,244 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:15,244 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:15,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:15,245 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:38405 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:04:15,245 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38405 2024-11-16T04:04:15,250 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=57 (was 57), OpenFileDescriptor=364 (was 364), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6846 (was 6847) 2024-11-16T04:04:15,255 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=57, OpenFileDescriptor=364, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6846 2024-11-16T04:04:15,256 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:15,256 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34707 2024-11-16T04:04:15,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bfa0172, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:15,256 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:15,256 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:15,272 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:15,284 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:15,284 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xef4b8c10, L:/127.0.0.1:34707 - R:/127.0.0.1:60708] 2024-11-16T04:04:15,284 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:15,366 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34707 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:15,476 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34707 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:15,587 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34707 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:15,697 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34707 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:15,807 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34707 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:15,916 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34707 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:04:16,026 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34707 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:16,136 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34707 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:16,176 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42652 deadline: 1731729855276 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42652 2024-11-16T04:04:16,176 WARN [TestNettyTlsIPC-1-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:16,176 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42652 deadline: 1731729855276 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42652 2024-11-16T04:04:16,177 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42652 deadline: 1731729855276 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42652 2024-11-16T04:04:16,177 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42652 deadline: 1731729855276 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42652 2024-11-16T04:04:16,177 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42652 deadline: 1731729855276 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42652 2024-11-16T04:04:16,177 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42652 deadline: 1731729855276 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42652 2024-11-16T04:04:16,177 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42652 deadline: 1731729855276 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42652 2024-11-16T04:04:16,177 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42652 deadline: 1731729855276 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42652 2024-11-16T04:04:16,178 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42652 deadline: 1731729855277 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42652 2024-11-16T04:04:16,246 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34707 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:16,285 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:60708 deadline: 1731729855467 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:60708 2024-11-16T04:04:16,286 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:04:16,286 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:60708 deadline: 1731729855578 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:60708 2024-11-16T04:04:16,286 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:60708 deadline: 1731729855688 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:60708 2024-11-16T04:04:16,287 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:60708 deadline: 1731729855798 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:60708 2024-11-16T04:04:16,287 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:60708 deadline: 1731729855907 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:60708 2024-11-16T04:04:16,287 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:60708 deadline: 1731729856016 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:60708 2024-11-16T04:04:16,288 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:60708 deadline: 1731729856127 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:60708 2024-11-16T04:04:16,288 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:60708 deadline: 1731729856237 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:60708 2024-11-16T04:04:16,356 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:34707 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:16,356 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:16,356 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34707 2024-11-16T04:04:16,361 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=57 (was 57), OpenFileDescriptor=364 (was 364), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6847 (was 6846) - AvailableMemoryMB LEAK? - 2024-11-16T04:04:16,366 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=57, OpenFileDescriptor=364, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6847 2024-11-16T04:04:16,367 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:16,367 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34453 2024-11-16T04:04:16,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16e1af49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:16,367 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:16,368 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:16,380 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:16,393 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:16,393 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x82aeb332, L:/127.0.0.1:34453 - R:/127.0.0.1:54864] 2024-11-16T04:04:16,393 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:16,394 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:16,394 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:54864 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:16,395 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:16,395 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:16,395 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:16,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:16,400 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=58 (was 57) - Thread LEAK? -, OpenFileDescriptor=365 (was 364) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6847 (was 6847) 2024-11-16T04:04:16,405 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=58, OpenFileDescriptor=365, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6847 2024-11-16T04:04:16,406 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:16,406 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45843 2024-11-16T04:04:16,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d3eb94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:16,406 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:16,407 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:16,425 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:16,437 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:16,437 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x8946752f, L:/127.0.0.1:45843 - R:/127.0.0.1:47272] 2024-11-16T04:04:16,437 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:16,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:16,444 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45843 2024-11-16T04:04:16,449 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=58 (was 58), OpenFileDescriptor=365 (was 365), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6846 (was 6847) 2024-11-16T04:04:16,454 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=58, OpenFileDescriptor=365, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6846 2024-11-16T04:04:16,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65a71c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:16,455 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:16,456 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33799 2024-11-16T04:04:16,456 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:16,456 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:16,475 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:16,487 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:16,487 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x72935bc8, L:/127.0.0.1:33799 - R:/127.0.0.1:55996] 2024-11-16T04:04:16,487 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:16,488 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:16,488 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:55996 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:16,489 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:16,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:16,489 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33799 2024-11-16T04:04:16,494 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=58 (was 58), OpenFileDescriptor=365 (was 365), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6846 (was 6846) 2024-11-16T04:04:16,498 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=58, OpenFileDescriptor=365, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6846 2024-11-16T04:04:16,499 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:16,500 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45917 2024-11-16T04:04:16,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a2bec2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:16,500 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:16,500 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:16,516 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:16,529 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:16,529 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x97e431d7, L:/127.0.0.1:45917 - R:/127.0.0.1:59892] 2024-11-16T04:04:16,529 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:16,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:16,531 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45917 2024-11-16T04:04:16,536 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=58 (was 58), OpenFileDescriptor=365 (was 365), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6846 (was 6846) 2024-11-16T04:04:16,541 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=58, OpenFileDescriptor=365, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6846 2024-11-16T04:04:16,542 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:16,543 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34561 2024-11-16T04:04:16,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5725836f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:16,543 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:16,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:16,560 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:16,572 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:16,572 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xfc3b03fd, L:/127.0.0.1:34561 - R:/127.0.0.1:60640] 2024-11-16T04:04:16,572 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:16,572 WARN [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:60640; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:16,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:16,573 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:34561 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:04:16,573 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34561 2024-11-16T04:04:16,578 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=58 (was 58), OpenFileDescriptor=365 (was 365), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6846 (was 6846) 2024-11-16T04:04:16,582 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=58, OpenFileDescriptor=365, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6845 2024-11-16T04:04:16,583 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:16,584 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43531 2024-11-16T04:04:16,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ad85a57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:16,584 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:16,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:16,596 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:16,609 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:16,609 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x4af1da95, L:/127.0.0.1:43531 - R:/127.0.0.1:42036] 2024-11-16T04:04:16,609 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:04:16,609 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:04:16,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:16,614 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=58 (was 58), OpenFileDescriptor=366 (was 365) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6846 (was 6845) - AvailableMemoryMB LEAK? - 2024-11-16T04:04:16,618 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=58, OpenFileDescriptor=366, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6846 2024-11-16T04:04:16,619 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:16,619 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42571 2024-11-16T04:04:16,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6c0e94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:16,620 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:16,620 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:16,635 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:16,647 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:16,647 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xf3f7a04d, L:/127.0.0.1:42571 - R:/127.0.0.1:43604] 2024-11-16T04:04:16,647 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:04:16,647 WARN [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:43604 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:43604 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:16,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:16,653 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=58 (was 58), OpenFileDescriptor=367 (was 366) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6846 (was 6846) 2024-11-16T04:04:16,657 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=58, OpenFileDescriptor=367, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6846 2024-11-16T04:04:16,658 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:16,658 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46435 2024-11-16T04:04:16,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@298e6ea0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:16,658 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:16,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:16,675 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:16,686 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:16,686 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x4859802f, L:/127.0.0.1:46435 - R:/127.0.0.1:58472] 2024-11-16T04:04:16,687 WARN [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:58472 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:58472 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:16,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:16,687 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46435 2024-11-16T04:04:16,692 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=58 (was 58), OpenFileDescriptor=367 (was 367), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6846 (was 6846) 2024-11-16T04:04:16,697 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=58, OpenFileDescriptor=367, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6846 2024-11-16T04:04:16,698 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:16,698 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39359 2024-11-16T04:04:16,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d1e5825, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:16,698 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:16,699 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:16,715 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:16,729 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:16,729 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x408260e4, L:/127.0.0.1:39359 - R:/127.0.0.1:48740] 2024-11-16T04:04:16,729 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:16,831 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:16,831 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:16,832 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:16,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:16,843 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=59 (was 58) - Thread LEAK? -, OpenFileDescriptor=368 (was 367) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6846 (was 6846) 2024-11-16T04:04:16,848 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=59, OpenFileDescriptor=368, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6846 2024-11-16T04:04:16,849 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:16,849 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36069 2024-11-16T04:04:16,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2490a578, compressor=org.apache.hadoop.io.compress.GzipCodec@722634d0, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:16,849 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:16,849 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:16,865 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:16,878 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:16,878 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x444fbbf0, L:/127.0.0.1:36069 - R:/127.0.0.1:37394] 2024-11-16T04:04:16,878 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:16,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:16,880 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36069 2024-11-16T04:04:16,885 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=59 (was 59), OpenFileDescriptor=368 (was 368), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6846 (was 6846) 2024-11-16T04:04:16,889 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=59, OpenFileDescriptor=368, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6846 2024-11-16T04:04:16,890 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:16,890 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36977 2024-11-16T04:04:16,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ac40877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:16,891 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:16,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:16,905 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:16,919 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:16,919 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x1fe38f18, L:/127.0.0.1:36977 - R:/127.0.0.1:40074] 2024-11-16T04:04:16,919 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:16,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:16,920 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36977 2024-11-16T04:04:16,926 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=59 (was 59), OpenFileDescriptor=368 (was 368), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6846 (was 6846) 2024-11-16T04:04:16,930 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=59, OpenFileDescriptor=368, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6846 2024-11-16T04:04:16,931 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:16,931 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33621 2024-11-16T04:04:16,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cb6e8a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:16,931 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:16,932 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:16,948 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:16,960 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:16,960 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x5030f5bf, L:/127.0.0.1:33621 - R:/127.0.0.1:36352] 2024-11-16T04:04:16,960 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:16,961 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:16,961 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:36352 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:16,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:16,961 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:16,961 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33621 2024-11-16T04:04:16,966 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=59 (was 59), OpenFileDescriptor=368 (was 368), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6846 (was 6846) 2024-11-16T04:04:16,970 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=59, OpenFileDescriptor=368, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6845 2024-11-16T04:04:16,972 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:16,972 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33477 2024-11-16T04:04:16,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:16,972 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:16,972 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:16,988 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:17,000 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:17,000 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x746a1556, L:/127.0.0.1:33477 - R:/127.0.0.1:44022] 2024-11-16T04:04:17,000 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:17,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:17,001 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33477 2024-11-16T04:04:17,006 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=59 (was 59), OpenFileDescriptor=368 (was 368), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6846 (was 6845) - AvailableMemoryMB LEAK? - 2024-11-16T04:04:17,011 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=59, OpenFileDescriptor=368, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6846 2024-11-16T04:04:17,012 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:17,012 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40511 2024-11-16T04:04:17,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a43498d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:17,012 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:17,013 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:17,030 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:17,042 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:17,042 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xa6086159, L:/127.0.0.1:40511 - R:/127.0.0.1:54676] 2024-11-16T04:04:17,042 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:17,042 WARN [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:17,042 INFO [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-42d22bd8adc3e597c7c88b394c131db3-6b6b97a3f5acc3a3-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:04:17,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:17,043 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:17,043 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40511 2024-11-16T04:04:17,048 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=59 (was 59), OpenFileDescriptor=368 (was 368), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6846 (was 6846) 2024-11-16T04:04:17,052 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=59, OpenFileDescriptor=368, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6845 2024-11-16T04:04:17,053 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:17,054 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35961 2024-11-16T04:04:17,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fcb93af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:17,054 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:17,054 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:17,071 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:17,084 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:17,084 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x58b31903, L:/127.0.0.1:35961 - R:/127.0.0.1:49260] 2024-11-16T04:04:17,084 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:17,167 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:35961 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=101ms, rpcTimeout=100ms 2024-11-16T04:04:17,167 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:35961 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=101ms, rpcTimeout=100ms 2024-11-16T04:04:17,167 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:35961 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=111ms, rpcTimeout=100ms 2024-11-16T04:04:17,167 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:35961 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=111ms, rpcTimeout=100ms 2024-11-16T04:04:17,167 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:35961 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=111ms, rpcTimeout=100ms 2024-11-16T04:04:17,167 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:35961 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=111ms, rpcTimeout=100ms 2024-11-16T04:04:17,167 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:35961 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=112ms, rpcTimeout=100ms 2024-11-16T04:04:17,167 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:35961 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=112ms, rpcTimeout=100ms 2024-11-16T04:04:17,167 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:35961 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=112ms, rpcTimeout=100ms 2024-11-16T04:04:17,168 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:35961 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=111ms, rpcTimeout=100ms 2024-11-16T04:04:17,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:17,168 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35961 2024-11-16T04:04:17,178 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[17: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=RSA, keyPassword=[C@64beebb7, acceptPlainText=true, clientTlsEnabled=false] Thread=60 (was 59) - Thread LEAK? -, OpenFileDescriptor=368 (was 368), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6845 (was 6845) 2024-11-16T04:04:17,187 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60, OpenFileDescriptor=368, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6845 2024-11-16T04:04:17,287 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:17,288 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33187 2024-11-16T04:04:17,288 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d2289e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:17,288 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:17,288 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:17,288 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:17,288 WARN [TestNettyTlsIPC-1-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:17,289 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:33187 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:04:17,289 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33187 2024-11-16T04:04:17,294 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=59 (was 60), OpenFileDescriptor=368 (was 368), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6845 (was 6845) 2024-11-16T04:04:17,298 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=59, OpenFileDescriptor=368, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6845 2024-11-16T04:04:17,299 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:17,300 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40477 2024-11-16T04:04:17,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4529287a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:17,300 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:17,301 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:17,301 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:17,331 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:17,331 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:17,344 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:17,356 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:17,356 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xfda2e839, L:/127.0.0.1:40477 - R:/127.0.0.1:40214] 2024-11-16T04:04:17,383 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:17,436 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40477 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=136ms, rpcTimeout=100ms 2024-11-16T04:04:17,546 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40477 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:17,657 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40477 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:17,767 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40477 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:17,877 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40477 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:17,986 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40477 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:18,085 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49260 deadline: 1731729857184 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49260 2024-11-16T04:04:18,085 WARN [TestNettyTlsIPC-1-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:18,085 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49260 deadline: 1731729857184 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49260 2024-11-16T04:04:18,085 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49260 deadline: 1731729857185 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49260 2024-11-16T04:04:18,085 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49260 deadline: 1731729857185 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49260 2024-11-16T04:04:18,086 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49260 deadline: 1731729857185 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49260 2024-11-16T04:04:18,086 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49260 deadline: 1731729857185 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49260 2024-11-16T04:04:18,086 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49260 deadline: 1731729857185 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49260 2024-11-16T04:04:18,086 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49260 deadline: 1731729857185 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49260 2024-11-16T04:04:18,086 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:49260 deadline: 1731729857185 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:49260 2024-11-16T04:04:18,096 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40477 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:18,206 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40477 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:18,317 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40477 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:18,384 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:40214 deadline: 1731729857536 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:40214 2024-11-16T04:04:18,385 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:40214 deadline: 1731729857647 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:40214 2024-11-16T04:04:18,385 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:04:18,385 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:40214 deadline: 1731729857758 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:40214 2024-11-16T04:04:18,385 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:40214 deadline: 1731729857868 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:40214 2024-11-16T04:04:18,385 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:40214 deadline: 1731729857978 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:40214 2024-11-16T04:04:18,385 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:40214 deadline: 1731729858087 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:40214 2024-11-16T04:04:18,385 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:40214 deadline: 1731729858197 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:40214 2024-11-16T04:04:18,385 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:40214 deadline: 1731729858307 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:40214 2024-11-16T04:04:18,426 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40477 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:18,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:18,427 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40477 2024-11-16T04:04:18,439 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=59 (was 59), OpenFileDescriptor=368 (was 368), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6845 (was 6845) 2024-11-16T04:04:18,444 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=59, OpenFileDescriptor=368, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6845 2024-11-16T04:04:18,445 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:18,445 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39415 2024-11-16T04:04:18,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff241d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:18,445 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:18,446 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:18,446 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:18,471 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:18,472 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:18,484 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:18,496 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:18,496 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xde5f3b6b, L:/127.0.0.1:39415 - R:/127.0.0.1:50826] 2024-11-16T04:04:18,523 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:18,524 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:18,524 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:50826 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:18,525 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:18,525 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:18,525 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:18,525 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:18,531 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60 (was 59) - Thread LEAK? -, OpenFileDescriptor=369 (was 368) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6845 (was 6845) 2024-11-16T04:04:18,535 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60, OpenFileDescriptor=369, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6845 2024-11-16T04:04:18,536 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:18,537 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34433 2024-11-16T04:04:18,537 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d82e942, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:18,537 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:18,537 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:18,537 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:18,562 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:18,563 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:18,575 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:18,587 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:18,588 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x70252316, L:/127.0.0.1:34433 - R:/127.0.0.1:46322] 2024-11-16T04:04:18,614 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:18,622 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:18,622 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34433 2024-11-16T04:04:18,627 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60 (was 60), OpenFileDescriptor=369 (was 369), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6845 (was 6845) 2024-11-16T04:04:18,632 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60, OpenFileDescriptor=369, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6845 2024-11-16T04:04:18,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c8a08e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:18,633 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:18,633 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43207 2024-11-16T04:04:18,633 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:18,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:18,634 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:18,659 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:18,659 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:18,671 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:18,683 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:18,684 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x49be83b5, L:/127.0.0.1:43207 - R:/127.0.0.1:35682] 2024-11-16T04:04:18,710 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:18,711 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:18,711 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:35682 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:18,711 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:18,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:18,712 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43207 2024-11-16T04:04:18,717 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60 (was 60), OpenFileDescriptor=369 (was 369), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6845 (was 6845) 2024-11-16T04:04:18,722 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60, OpenFileDescriptor=369, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6845 2024-11-16T04:04:18,723 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:18,723 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44807 2024-11-16T04:04:18,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f88f32d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:18,723 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:18,723 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:18,724 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:18,748 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:18,748 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:18,761 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:18,773 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:18,773 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xe3676f17, L:/127.0.0.1:44807 - R:/127.0.0.1:35122] 2024-11-16T04:04:18,800 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:18,802 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:18,802 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44807 2024-11-16T04:04:18,807 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60 (was 60), OpenFileDescriptor=369 (was 369), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6845 (was 6845) 2024-11-16T04:04:18,813 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60, OpenFileDescriptor=369, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6845 2024-11-16T04:04:18,813 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:18,814 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45633 2024-11-16T04:04:18,814 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a8151b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:18,814 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:18,814 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:18,814 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:18,839 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:18,839 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:18,852 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:18,864 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:18,864 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xb1cba597, L:/127.0.0.1:45633 - R:/127.0.0.1:60550] 2024-11-16T04:04:18,891 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:18,891 WARN [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:60550; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:18,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:18,892 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:45633 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:04:18,892 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45633 2024-11-16T04:04:18,897 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60 (was 60), OpenFileDescriptor=369 (was 369), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6845 (was 6845) 2024-11-16T04:04:18,902 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60, OpenFileDescriptor=369, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6845 2024-11-16T04:04:18,903 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:18,903 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39083 2024-11-16T04:04:18,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24688942, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:18,903 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:18,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:18,904 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:18,928 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:18,928 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:18,941 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:18,953 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:18,953 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xa258b298, L:/127.0.0.1:39083 - R:/127.0.0.1:58292] 2024-11-16T04:04:18,980 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:04:18,980 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:04:18,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:18,985 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60 (was 60), OpenFileDescriptor=370 (was 369) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6844 (was 6845) 2024-11-16T04:04:18,990 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60, OpenFileDescriptor=370, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6844 2024-11-16T04:04:18,991 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:18,991 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38645 2024-11-16T04:04:18,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6865fd1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:18,991 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:18,992 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:18,992 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:19,020 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:19,020 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:19,034 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:19,046 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:19,046 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xe6c8aae6, L:/127.0.0.1:38645 - R:/127.0.0.1:34376] 2024-11-16T04:04:19,072 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:04:19,072 WARN [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:34376 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:34376 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:19,073 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:19,078 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60 (was 60), OpenFileDescriptor=371 (was 370) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6845 (was 6844) - AvailableMemoryMB LEAK? - 2024-11-16T04:04:19,083 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60, OpenFileDescriptor=371, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6845 2024-11-16T04:04:19,084 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:19,084 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46409 2024-11-16T04:04:19,084 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3ba4ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:19,084 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:19,085 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:19,085 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:19,114 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:19,114 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:19,127 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:19,140 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:19,140 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x7ec6f87f, L:/127.0.0.1:46409 - R:/127.0.0.1:49068] 2024-11-16T04:04:19,167 WARN [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:49068 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:49068 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:19,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:19,168 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46409 2024-11-16T04:04:19,173 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60 (was 60), OpenFileDescriptor=371 (was 371), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=6845 (was 6845) 2024-11-16T04:04:19,178 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60, OpenFileDescriptor=371, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=6845 2024-11-16T04:04:19,178 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:19,179 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39037 2024-11-16T04:04:19,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51ed5dbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:19,179 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:19,179 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:19,180 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:19,204 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:19,204 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:19,217 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:19,229 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:19,229 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xf76dfda4, L:/127.0.0.1:39037 - R:/127.0.0.1:42564] 2024-11-16T04:04:19,256 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:19,358 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:19,358 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:19,358 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:19,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:19,364 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=61 (was 60) - Thread LEAK? -, OpenFileDescriptor=372 (was 371) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 130) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6841 (was 6845) 2024-11-16T04:04:19,369 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=61, OpenFileDescriptor=372, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6841 2024-11-16T04:04:19,370 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:19,370 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36297 2024-11-16T04:04:19,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da1b6f3, compressor=org.apache.hadoop.io.compress.GzipCodec@13637d31, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:19,371 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:19,371 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:19,371 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:19,385 WARN [TestNettyTlsIPC-1-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:19,396 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:19,396 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:19,409 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:19,421 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:19,422 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xa0f00691, L:/127.0.0.1:36297 - R:/127.0.0.1:42682] 2024-11-16T04:04:19,448 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:19,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:19,449 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36297 2024-11-16T04:04:19,455 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60 (was 61), OpenFileDescriptor=372 (was 372), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6840 (was 6841) 2024-11-16T04:04:19,461 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60, OpenFileDescriptor=372, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6840 2024-11-16T04:04:19,462 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:19,462 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43141 2024-11-16T04:04:19,463 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d102632, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:19,463 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:19,463 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:19,463 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:19,491 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:19,491 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:19,503 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:19,516 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:19,516 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x1eb2e649, L:/127.0.0.1:43141 - R:/127.0.0.1:59828] 2024-11-16T04:04:19,544 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:19,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:19,545 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43141 2024-11-16T04:04:19,551 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60 (was 60), OpenFileDescriptor=372 (was 372), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6840 (was 6840) 2024-11-16T04:04:19,558 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60, OpenFileDescriptor=372, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6840 2024-11-16T04:04:19,559 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:19,559 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35549 2024-11-16T04:04:19,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42070a77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:19,559 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:19,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:19,560 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:19,585 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:19,586 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:19,600 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:19,613 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:19,613 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x13605c4f, L:/127.0.0.1:35549 - R:/127.0.0.1:43832] 2024-11-16T04:04:19,643 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:19,643 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:19,643 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:43832 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:19,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:19,644 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:19,644 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35549 2024-11-16T04:04:19,650 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60 (was 60), OpenFileDescriptor=372 (was 372), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6840 (was 6840) 2024-11-16T04:04:19,654 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60, OpenFileDescriptor=372, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6840 2024-11-16T04:04:19,655 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:19,655 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40147 2024-11-16T04:04:19,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:19,656 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:19,656 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:19,656 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:19,683 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:19,684 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:19,698 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:19,710 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:19,711 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xb5673285, L:/127.0.0.1:40147 - R:/127.0.0.1:49320] 2024-11-16T04:04:19,738 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:19,739 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:19,739 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40147 2024-11-16T04:04:19,745 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60 (was 60), OpenFileDescriptor=372 (was 372), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6840 (was 6840) 2024-11-16T04:04:19,750 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60, OpenFileDescriptor=372, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6840 2024-11-16T04:04:19,750 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:19,751 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39873 2024-11-16T04:04:19,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44b10d85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:19,751 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:19,751 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:19,751 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:19,778 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:19,778 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:19,790 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:19,803 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:19,803 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xc7d99f94, L:/127.0.0.1:39873 - R:/127.0.0.1:57946] 2024-11-16T04:04:19,833 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:19,833 WARN [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:19,834 INFO [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-0616f06396ea0354eecd3677bd3cfb41-86877c4abfb2b233-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:04:19,834 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:19,834 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:19,834 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39873 2024-11-16T04:04:19,839 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60 (was 60), OpenFileDescriptor=372 (was 372), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6840 (was 6840) 2024-11-16T04:04:19,845 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=60, OpenFileDescriptor=372, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6840 2024-11-16T04:04:19,846 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:19,847 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45647 2024-11-16T04:04:19,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2204e97c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:19,847 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:19,847 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:19,848 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:19,879 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:19,879 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:19,892 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:19,904 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:19,904 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xbeef2b17, L:/127.0.0.1:45647 - R:/127.0.0.1:54458] 2024-11-16T04:04:19,932 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:19,987 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45647 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=139ms, rpcTimeout=100ms 2024-11-16T04:04:19,987 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45647 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=139ms, rpcTimeout=100ms 2024-11-16T04:04:19,987 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45647 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=138ms, rpcTimeout=100ms 2024-11-16T04:04:19,987 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45647 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=138ms, rpcTimeout=100ms 2024-11-16T04:04:19,987 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45647 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=138ms, rpcTimeout=100ms 2024-11-16T04:04:19,987 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45647 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=139ms, rpcTimeout=100ms 2024-11-16T04:04:19,987 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45647 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=139ms, rpcTimeout=100ms 2024-11-16T04:04:19,987 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45647 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=139ms, rpcTimeout=100ms 2024-11-16T04:04:19,987 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45647 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=139ms, rpcTimeout=100ms 2024-11-16T04:04:19,987 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:45647 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=139ms, rpcTimeout=100ms 2024-11-16T04:04:19,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:19,987 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45647 2024-11-16T04:04:19,997 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[18: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=false, clientTlsEnabled=true] Thread=61 (was 60) - Thread LEAK? -, OpenFileDescriptor=372 (was 372), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6840 (was 6840) 2024-11-16T04:04:20,004 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=61, OpenFileDescriptor=372, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6840 2024-11-16T04:04:20,005 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:20,005 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33487 2024-11-16T04:04:20,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3761b452, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:20,005 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:20,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:20,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:20,006 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:33487 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:04:20,006 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33487 2024-11-16T04:04:20,011 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=61 (was 61), OpenFileDescriptor=372 (was 372), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6839 (was 6840) 2024-11-16T04:04:20,015 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=61, OpenFileDescriptor=372, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6839 2024-11-16T04:04:20,016 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:20,017 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44947 2024-11-16T04:04:20,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e18ed9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:20,017 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:20,017 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:20,017 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:20,044 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:20,045 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:20,057 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:20,070 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:20,071 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xc768365b, L:/127.0.0.1:44947 - R:/127.0.0.1:48880] 2024-11-16T04:04:20,097 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:20,146 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44947 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:04:20,256 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44947 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:20,367 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44947 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:20,477 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44947 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:20,587 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44947 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:20,696 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44947 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:20,807 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44947 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:20,917 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44947 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:20,933 WARN [TestNettyTlsIPC-1-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:20,933 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:54458 deadline: 1731729860032 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:54458 2024-11-16T04:04:20,933 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:54458 deadline: 1731729860032 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:54458 2024-11-16T04:04:20,933 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:54458 deadline: 1731729860033 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:54458 2024-11-16T04:04:20,933 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:54458 deadline: 1731729860033 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:54458 2024-11-16T04:04:20,933 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:54458 deadline: 1731729860033 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:54458 2024-11-16T04:04:20,933 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:54458 deadline: 1731729860033 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:54458 2024-11-16T04:04:20,933 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:54458 deadline: 1731729860033 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:54458 2024-11-16T04:04:20,933 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:54458 deadline: 1731729860033 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:54458 2024-11-16T04:04:20,933 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:54458 deadline: 1731729860033 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:54458 2024-11-16T04:04:21,027 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44947 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:21,099 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:48880 deadline: 1731729860247 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:48880 2024-11-16T04:04:21,099 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:48880 deadline: 1731729860357 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:48880 2024-11-16T04:04:21,099 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:04:21,099 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:48880 deadline: 1731729860468 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:48880 2024-11-16T04:04:21,099 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:48880 deadline: 1731729860578 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:48880 2024-11-16T04:04:21,100 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:48880 deadline: 1731729860688 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:48880 2024-11-16T04:04:21,100 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:48880 deadline: 1731729860797 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:48880 2024-11-16T04:04:21,100 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:48880 deadline: 1731729860908 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:48880 2024-11-16T04:04:21,100 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:48880 deadline: 1731729861018 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:48880 2024-11-16T04:04:21,137 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44947 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:21,137 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:21,137 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44947 2024-11-16T04:04:21,151 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=61 (was 61), OpenFileDescriptor=372 (was 372), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6840 (was 6839) - AvailableMemoryMB LEAK? - 2024-11-16T04:04:21,160 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=61, OpenFileDescriptor=372, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6839 2024-11-16T04:04:21,163 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:21,164 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46415 2024-11-16T04:04:21,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f418ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:21,164 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:21,164 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:21,165 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:21,193 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:21,193 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:21,206 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:21,218 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:21,219 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xf5998462, L:/127.0.0.1:46415 - R:/127.0.0.1:38830] 2024-11-16T04:04:21,245 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:21,246 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:21,246 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:38830 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:21,247 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:21,247 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:21,247 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:21,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:21,252 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62 (was 61) - Thread LEAK? -, OpenFileDescriptor=373 (was 372) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6839 (was 6839) 2024-11-16T04:04:21,258 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62, OpenFileDescriptor=373, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6839 2024-11-16T04:04:21,259 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:21,259 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44855 2024-11-16T04:04:21,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@212332a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:21,260 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:21,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:21,260 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:21,286 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:21,286 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:21,300 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:21,311 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:21,312 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x2792969d, L:/127.0.0.1:44855 - R:/127.0.0.1:58486] 2024-11-16T04:04:21,339 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:21,346 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:21,347 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44855 2024-11-16T04:04:21,352 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62 (was 62), OpenFileDescriptor=373 (was 373), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6839 (was 6839) 2024-11-16T04:04:21,357 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62, OpenFileDescriptor=373, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6839 2024-11-16T04:04:21,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a320f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:21,358 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:21,358 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43079 2024-11-16T04:04:21,358 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:21,359 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:21,359 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:21,383 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:21,384 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:21,397 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:21,409 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:21,409 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x31dacc58, L:/127.0.0.1:43079 - R:/127.0.0.1:57950] 2024-11-16T04:04:21,437 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:21,438 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:21,438 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:57950 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:21,439 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:21,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:21,439 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43079 2024-11-16T04:04:21,444 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62 (was 62), OpenFileDescriptor=373 (was 373), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6839 (was 6839) 2024-11-16T04:04:21,449 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62, OpenFileDescriptor=373, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6839 2024-11-16T04:04:21,450 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:21,450 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35259 2024-11-16T04:04:21,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35ed6ea5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:21,450 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:21,451 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:21,451 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:21,481 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:21,481 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:21,493 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:21,505 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:21,506 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x0a62b29b, L:/127.0.0.1:35259 - R:/127.0.0.1:34180] 2024-11-16T04:04:21,533 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:21,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:21,536 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35259 2024-11-16T04:04:21,541 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62 (was 62), OpenFileDescriptor=373 (was 373), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6839 (was 6839) 2024-11-16T04:04:21,546 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62, OpenFileDescriptor=373, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6839 2024-11-16T04:04:21,547 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:21,547 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36471 2024-11-16T04:04:21,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19af9eb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:21,547 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:21,548 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:21,548 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:21,576 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:21,576 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:21,588 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:21,601 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:21,601 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xef2ca706, L:/127.0.0.1:36471 - R:/127.0.0.1:34836] 2024-11-16T04:04:21,629 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:21,629 WARN [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:34836; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:21,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:21,629 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:36471 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:04:21,629 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36471 2024-11-16T04:04:21,634 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62 (was 62), OpenFileDescriptor=373 (was 373), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6839 (was 6839) 2024-11-16T04:04:21,639 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62, OpenFileDescriptor=373, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6839 2024-11-16T04:04:21,641 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:21,641 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43949 2024-11-16T04:04:21,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7037776f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:21,641 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:21,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:21,642 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:21,666 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:21,666 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:21,690 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:21,708 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:21,708 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xd57ac363, L:/127.0.0.1:43949 - R:/127.0.0.1:60110] 2024-11-16T04:04:21,735 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:04:21,735 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:04:21,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:21,741 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62 (was 62), OpenFileDescriptor=374 (was 373) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6839 (was 6839) 2024-11-16T04:04:21,745 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62, OpenFileDescriptor=374, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6839 2024-11-16T04:04:21,746 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:21,747 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46823 2024-11-16T04:04:21,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556dad71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:21,747 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:21,747 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:21,747 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:21,772 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:21,772 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:21,785 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:21,797 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:21,798 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x6be6ad96, L:/127.0.0.1:46823 - R:/127.0.0.1:44452] 2024-11-16T04:04:21,823 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:04:21,823 WARN [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:44452 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:44452 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:21,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:21,829 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62 (was 62), OpenFileDescriptor=375 (was 374) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6839 (was 6839) 2024-11-16T04:04:21,834 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62, OpenFileDescriptor=375, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6839 2024-11-16T04:04:21,835 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:21,835 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46291 2024-11-16T04:04:21,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d924f63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:21,835 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:21,836 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:21,836 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:21,862 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:21,862 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:21,886 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:21,900 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:21,901 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x0e026d0f, L:/127.0.0.1:46291 - R:/127.0.0.1:38244] 2024-11-16T04:04:21,928 WARN [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:38244 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:38244 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:21,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:21,929 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46291 2024-11-16T04:04:21,934 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62 (was 62), OpenFileDescriptor=375 (was 375), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6839 (was 6839) 2024-11-16T04:04:21,939 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62, OpenFileDescriptor=375, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6839 2024-11-16T04:04:21,940 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:21,940 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40723 2024-11-16T04:04:21,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1acb89eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:21,940 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:21,941 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:21,941 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:21,966 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:21,966 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:21,979 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:21,991 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:21,991 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xd8f14d21, L:/127.0.0.1:40723 - R:/127.0.0.1:37246] 2024-11-16T04:04:22,018 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:22,101 WARN [TestNettyTlsIPC-1-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:22,119 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:22,120 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:22,120 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:22,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:22,130 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62 (was 62), OpenFileDescriptor=376 (was 375) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6839 (was 6839) 2024-11-16T04:04:22,137 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62, OpenFileDescriptor=376, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6839 2024-11-16T04:04:22,138 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:22,138 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36239 2024-11-16T04:04:22,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cb4d06, compressor=org.apache.hadoop.io.compress.GzipCodec@2bf97286, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:22,138 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:22,138 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:22,139 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:22,167 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:22,167 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:22,181 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:22,193 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:22,193 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x145957c7, L:/127.0.0.1:36239 - R:/127.0.0.1:38166] 2024-11-16T04:04:22,220 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:22,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:22,222 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36239 2024-11-16T04:04:22,227 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62 (was 62), OpenFileDescriptor=376 (was 376), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6839 (was 6839) 2024-11-16T04:04:22,231 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62, OpenFileDescriptor=376, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6838 2024-11-16T04:04:22,232 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:22,233 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38947 2024-11-16T04:04:22,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e28b94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:22,233 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:22,233 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:22,233 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:22,260 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:22,260 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:22,273 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:22,285 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:22,285 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x08981d24, L:/127.0.0.1:38947 - R:/127.0.0.1:55564] 2024-11-16T04:04:22,311 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:22,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:22,312 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38947 2024-11-16T04:04:22,318 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62 (was 62), OpenFileDescriptor=376 (was 376), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6838 (was 6838) 2024-11-16T04:04:22,323 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62, OpenFileDescriptor=376, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6838 2024-11-16T04:04:22,324 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:22,324 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38823 2024-11-16T04:04:22,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69255529, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:22,325 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:22,325 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:22,325 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:22,355 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:22,355 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:22,368 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:22,381 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:22,381 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x33555a65, L:/127.0.0.1:38823 - R:/127.0.0.1:46262] 2024-11-16T04:04:22,407 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:22,408 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:22,408 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:46262 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:22,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:22,409 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:22,409 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38823 2024-11-16T04:04:22,414 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62 (was 62), OpenFileDescriptor=376 (was 376), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6838 (was 6838) 2024-11-16T04:04:22,420 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62, OpenFileDescriptor=376, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6838 2024-11-16T04:04:22,421 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:22,421 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45455 2024-11-16T04:04:22,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:22,421 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:22,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:22,422 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:22,451 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:22,451 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:22,465 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:22,477 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:22,477 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x44e1269a, L:/127.0.0.1:45455 - R:/127.0.0.1:38094] 2024-11-16T04:04:22,503 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:22,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:22,504 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45455 2024-11-16T04:04:22,509 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62 (was 62), OpenFileDescriptor=376 (was 376), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6844 (was 6838) - AvailableMemoryMB LEAK? - 2024-11-16T04:04:22,515 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62, OpenFileDescriptor=376, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6844 2024-11-16T04:04:22,516 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:22,516 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39623 2024-11-16T04:04:22,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d956d64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:22,516 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:22,517 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:22,517 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:22,547 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:22,547 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:22,560 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:22,573 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:22,573 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xa04b5010, L:/127.0.0.1:39623 - R:/127.0.0.1:34208] 2024-11-16T04:04:22,600 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:22,600 WARN [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:22,600 INFO [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-b49dadc7314ba4b64d2401abcfe06319-eea0ec7c9656815b-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:04:22,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:22,600 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:22,601 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39623 2024-11-16T04:04:22,605 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62 (was 62), OpenFileDescriptor=376 (was 376), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6844 (was 6844) 2024-11-16T04:04:22,610 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=62, OpenFileDescriptor=376, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6844 2024-11-16T04:04:22,611 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:22,611 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33667 2024-11-16T04:04:22,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@736bf265, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:22,611 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:22,611 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:22,612 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:22,639 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:22,639 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:22,652 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:22,665 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:22,665 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x6f20e5d8, L:/127.0.0.1:33667 - R:/127.0.0.1:38898] 2024-11-16T04:04:22,694 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:22,747 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33667 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:04:22,747 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33667 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:04:22,747 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33667 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:04:22,747 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33667 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:04:22,747 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33667 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:04:22,747 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33667 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:04:22,747 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33667 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:04:22,747 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33667 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=134ms, rpcTimeout=100ms 2024-11-16T04:04:22,747 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33667 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:04:22,747 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33667 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=135ms, rpcTimeout=100ms 2024-11-16T04:04:22,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:22,747 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33667 2024-11-16T04:04:22,757 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[19: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=true] Thread=63 (was 62) - Thread LEAK? -, OpenFileDescriptor=376 (was 376), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6844 (was 6844) 2024-11-16T04:04:22,767 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=63, OpenFileDescriptor=376, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6844 2024-11-16T04:04:22,768 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:22,768 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45691 2024-11-16T04:04:22,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e05992, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:22,768 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:22,768 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:22,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:22,769 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:45691 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:04:22,769 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45691 2024-11-16T04:04:22,773 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=63 (was 63), OpenFileDescriptor=376 (was 376), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6844 (was 6844) 2024-11-16T04:04:22,778 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=63, OpenFileDescriptor=376, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6844 2024-11-16T04:04:22,779 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:22,779 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33533 2024-11-16T04:04:22,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4af2d6da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:22,779 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:22,780 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:22,798 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:22,810 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:22,810 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x8d739263, L:/127.0.0.1:33533 - R:/127.0.0.1:52430] 2024-11-16T04:04:22,810 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:52430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:22,886 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33533 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=106ms, rpcTimeout=100ms 2024-11-16T04:04:22,996 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33533 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:23,106 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33533 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:23,216 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33533 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:23,327 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33533 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:23,436 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33533 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:23,546 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33533 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:23,656 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33533 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:23,696 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:38898 deadline: 1731729862795 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:38898 2024-11-16T04:04:23,696 WARN [TestNettyTlsIPC-1-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:23,696 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:38898 deadline: 1731729862795 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:38898 2024-11-16T04:04:23,696 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:38898 deadline: 1731729862795 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:38898 2024-11-16T04:04:23,696 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:38898 deadline: 1731729862795 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:38898 2024-11-16T04:04:23,696 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:38898 deadline: 1731729862795 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:38898 2024-11-16T04:04:23,697 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:38898 deadline: 1731729862795 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:38898 2024-11-16T04:04:23,697 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:38898 deadline: 1731729862796 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:38898 2024-11-16T04:04:23,697 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:38898 deadline: 1731729862796 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:38898 2024-11-16T04:04:23,697 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:38898 deadline: 1731729862796 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:38898 2024-11-16T04:04:23,766 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33533 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:04:23,811 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52430 deadline: 1731729862987 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52430 2024-11-16T04:04:23,811 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:04:23,811 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52430 deadline: 1731729863097 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52430 2024-11-16T04:04:23,811 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52430 deadline: 1731729863207 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52430 2024-11-16T04:04:23,811 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52430 deadline: 1731729863317 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52430 2024-11-16T04:04:23,812 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52430 deadline: 1731729863427 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52430 2024-11-16T04:04:23,812 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52430 deadline: 1731729863538 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52430 2024-11-16T04:04:23,812 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52430 deadline: 1731729863648 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52430 2024-11-16T04:04:23,812 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:52430 deadline: 1731729863758 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:52430 2024-11-16T04:04:23,876 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:33533 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:23,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:23,876 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33533 2024-11-16T04:04:23,884 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=63 (was 63), OpenFileDescriptor=376 (was 376), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6844 (was 6844) 2024-11-16T04:04:23,891 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=63, OpenFileDescriptor=376, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6844 2024-11-16T04:04:23,893 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:23,893 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35375 2024-11-16T04:04:23,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ccd295, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:23,893 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:23,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:23,911 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:23,924 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:23,924 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x8db2bc98, L:/127.0.0.1:35375 - R:/127.0.0.1:40046] 2024-11-16T04:04:23,924 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:23,925 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:23,925 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:40046 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:23,925 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:23,925 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:23,926 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:23,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:23,931 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=64 (was 63) - Thread LEAK? -, OpenFileDescriptor=377 (was 376) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6843 (was 6844) 2024-11-16T04:04:23,935 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=64, OpenFileDescriptor=377, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6843 2024-11-16T04:04:23,937 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:23,937 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37177 2024-11-16T04:04:23,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b7d3857, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:23,937 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:23,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:23,953 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:23,966 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:23,966 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xecee6cec, L:/127.0.0.1:37177 - R:/127.0.0.1:35856] 2024-11-16T04:04:23,966 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:23,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:23,972 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37177 2024-11-16T04:04:23,977 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=64 (was 64), OpenFileDescriptor=377 (was 377), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6843 (was 6843) 2024-11-16T04:04:23,982 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=64, OpenFileDescriptor=377, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6843 2024-11-16T04:04:23,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b80d905, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:23,983 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:23,983 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38303 2024-11-16T04:04:23,983 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:23,984 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:23,997 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:24,009 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:24,009 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x6383e2c1, L:/127.0.0.1:38303 - R:/127.0.0.1:43300] 2024-11-16T04:04:24,010 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:24,010 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:24,010 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:43300 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:24,011 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:24,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:24,011 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38303 2024-11-16T04:04:24,016 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=64 (was 64), OpenFileDescriptor=377 (was 377), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6843 (was 6843) 2024-11-16T04:04:24,021 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=64, OpenFileDescriptor=377, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6843 2024-11-16T04:04:24,022 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:24,022 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43623 2024-11-16T04:04:24,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15722d13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:24,022 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:24,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:24,039 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:24,052 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:24,052 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xe8b0e37e, L:/127.0.0.1:43623 - R:/127.0.0.1:44058] 2024-11-16T04:04:24,052 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:24,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:24,054 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43623 2024-11-16T04:04:24,059 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=64 (was 64), OpenFileDescriptor=377 (was 377), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6843 (was 6843) 2024-11-16T04:04:24,064 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=64, OpenFileDescriptor=377, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6843 2024-11-16T04:04:24,064 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:24,065 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39739 2024-11-16T04:04:24,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53c60db2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:24,065 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:24,065 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:24,080 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:24,092 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:24,092 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xad09fc31, L:/127.0.0.1:39739 - R:/127.0.0.1:44832] 2024-11-16T04:04:24,092 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:24,092 WARN [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:44832; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:24,093 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:24,093 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:39739 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:04:24,093 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39739 2024-11-16T04:04:24,097 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=64 (was 64), OpenFileDescriptor=377 (was 377), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6842 (was 6843) 2024-11-16T04:04:24,102 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=64, OpenFileDescriptor=377, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6842 2024-11-16T04:04:24,103 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:24,103 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41025 2024-11-16T04:04:24,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055ddb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:24,104 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:24,104 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:24,120 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:24,132 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:24,132 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x6a5145fa, L:/127.0.0.1:41025 - R:/127.0.0.1:41794] 2024-11-16T04:04:24,132 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:04:24,133 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:04:24,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:24,138 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=64 (was 64), OpenFileDescriptor=378 (was 377) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6843 (was 6842) - AvailableMemoryMB LEAK? - 2024-11-16T04:04:24,143 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=64, OpenFileDescriptor=378, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6843 2024-11-16T04:04:24,143 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:24,144 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37073 2024-11-16T04:04:24,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3894f5b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:24,144 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:24,144 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:24,160 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:24,172 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:24,172 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x5c537b5e, L:/127.0.0.1:37073 - R:/127.0.0.1:39606] 2024-11-16T04:04:24,172 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:04:24,172 WARN [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:39606 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:39606 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:24,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:24,177 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=64 (was 64), OpenFileDescriptor=379 (was 378) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6843 (was 6843) 2024-11-16T04:04:24,182 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=64, OpenFileDescriptor=379, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6843 2024-11-16T04:04:24,183 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:24,183 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42519 2024-11-16T04:04:24,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac97c1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:24,183 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:24,184 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:24,199 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:24,210 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:24,211 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x8998af06, L:/127.0.0.1:42519 - R:/127.0.0.1:37830] 2024-11-16T04:04:24,211 WARN [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:37830 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:37830 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:24,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:24,211 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42519 2024-11-16T04:04:24,216 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=64 (was 64), OpenFileDescriptor=379 (was 379), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6842 (was 6843) 2024-11-16T04:04:24,221 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=64, OpenFileDescriptor=379, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=6842 2024-11-16T04:04:24,222 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:24,222 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34387 2024-11-16T04:04:24,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2144744c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:24,222 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:24,223 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:24,238 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:24,251 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:24,251 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x534ac06c, L:/127.0.0.1:34387 - R:/127.0.0.1:51136] 2024-11-16T04:04:24,251 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:24,353 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:24,353 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:24,354 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:24,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:24,365 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=65 (was 64) - Thread LEAK? -, OpenFileDescriptor=380 (was 379) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=6843 (was 6842) - AvailableMemoryMB LEAK? - 2024-11-16T04:04:24,370 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=65, OpenFileDescriptor=380, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6843 2024-11-16T04:04:24,370 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:24,371 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42923 2024-11-16T04:04:24,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@280c7492, compressor=org.apache.hadoop.io.compress.GzipCodec@6a8bea0d, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:24,371 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:24,371 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:24,384 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:24,397 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:24,397 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x239dd6d6, L:/127.0.0.1:42923 - R:/127.0.0.1:48826] 2024-11-16T04:04:24,397 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:24,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:24,399 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42923 2024-11-16T04:04:24,404 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=65 (was 65), OpenFileDescriptor=380 (was 380), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6843 (was 6843) 2024-11-16T04:04:24,408 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=65, OpenFileDescriptor=380, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6843 2024-11-16T04:04:24,409 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:24,409 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42363 2024-11-16T04:04:24,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31ee5e5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:24,410 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:24,410 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:24,426 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:24,438 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:24,438 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x1f2b86a3, L:/127.0.0.1:42363 - R:/127.0.0.1:50194] 2024-11-16T04:04:24,438 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:24,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:24,439 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42363 2024-11-16T04:04:24,444 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=65 (was 65), OpenFileDescriptor=380 (was 380), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6842 (was 6843) 2024-11-16T04:04:24,449 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=65, OpenFileDescriptor=380, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6842 2024-11-16T04:04:24,450 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:24,450 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42295 2024-11-16T04:04:24,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d9d98a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:24,450 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:24,450 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:24,464 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:24,476 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:24,476 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xe2a08c4b, L:/127.0.0.1:42295 - R:/127.0.0.1:39440] 2024-11-16T04:04:24,476 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:24,477 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:24,477 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:39440 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:24,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:24,477 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:24,477 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42295 2024-11-16T04:04:24,482 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=65 (was 65), OpenFileDescriptor=380 (was 380), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6843 (was 6842) - AvailableMemoryMB LEAK? - 2024-11-16T04:04:24,487 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=65, OpenFileDescriptor=380, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6842 2024-11-16T04:04:24,488 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:24,488 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38591 2024-11-16T04:04:24,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:24,489 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:24,489 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:24,505 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:24,517 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:24,518 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xe440c8dc, L:/127.0.0.1:38591 - R:/127.0.0.1:55640] 2024-11-16T04:04:24,518 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:24,519 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:24,519 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38591 2024-11-16T04:04:24,524 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=65 (was 65), OpenFileDescriptor=380 (was 380), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6842 (was 6842) 2024-11-16T04:04:24,529 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=65, OpenFileDescriptor=380, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6842 2024-11-16T04:04:24,530 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:24,530 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40979 2024-11-16T04:04:24,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a807318, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:24,530 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:24,530 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:24,547 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:24,559 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:24,559 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x0de42532, L:/127.0.0.1:40979 - R:/127.0.0.1:33794] 2024-11-16T04:04:24,559 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:24,559 WARN [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:24,560 INFO [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-a83d360be52d901c9071576f5b62aa75-d1028738c7654829-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:04:24,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:24,560 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:24,560 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40979 2024-11-16T04:04:24,565 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=65 (was 65), OpenFileDescriptor=380 (was 380), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6842 (was 6842) 2024-11-16T04:04:24,569 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=65, OpenFileDescriptor=380, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6842 2024-11-16T04:04:24,570 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:24,571 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36573 2024-11-16T04:04:24,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e2f8b29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:24,571 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:24,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:24,586 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:24,598 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:24,598 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x413845a8, L:/127.0.0.1:36573 - R:/127.0.0.1:53404] 2024-11-16T04:04:24,598 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:24,677 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36573 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=105ms, rpcTimeout=100ms 2024-11-16T04:04:24,677 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36573 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=105ms, rpcTimeout=100ms 2024-11-16T04:04:24,677 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36573 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=105ms, rpcTimeout=100ms 2024-11-16T04:04:24,677 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36573 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=104ms, rpcTimeout=100ms 2024-11-16T04:04:24,677 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36573 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=104ms, rpcTimeout=100ms 2024-11-16T04:04:24,677 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36573 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=104ms, rpcTimeout=100ms 2024-11-16T04:04:24,677 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36573 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=105ms, rpcTimeout=100ms 2024-11-16T04:04:24,677 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36573 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=105ms, rpcTimeout=100ms 2024-11-16T04:04:24,677 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36573 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=105ms, rpcTimeout=100ms 2024-11-16T04:04:24,677 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36573 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=105ms, rpcTimeout=100ms 2024-11-16T04:04:24,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:24,677 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36573 2024-11-16T04:04:24,688 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[20: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@7813cb11, acceptPlainText=true, clientTlsEnabled=false] Thread=66 (was 65) - Thread LEAK? -, OpenFileDescriptor=380 (was 380), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6842 (was 6842) 2024-11-16T04:04:24,693 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66, OpenFileDescriptor=380, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6842 2024-11-16T04:04:24,786 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:24,786 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39599 2024-11-16T04:04:24,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11fa2d1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:24,787 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:24,787 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:24,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:24,787 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:39599 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:04:24,787 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39599 2024-11-16T04:04:24,792 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66 (was 66), OpenFileDescriptor=380 (was 380), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6843 (was 6842) - AvailableMemoryMB LEAK? - 2024-11-16T04:04:24,797 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66, OpenFileDescriptor=380, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6843 2024-11-16T04:04:24,798 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:24,798 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39785 2024-11-16T04:04:24,798 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c651407, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:24,798 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:24,798 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:24,799 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:24,812 WARN [TestNettyTlsIPC-1-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:24,826 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:24,826 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:24,839 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:24,851 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:24,851 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xaa2cdfa9, L:/127.0.0.1:39785 - R:/127.0.0.1:51022] 2024-11-16T04:04:24,878 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:24,936 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39785 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=138ms, rpcTimeout=100ms 2024-11-16T04:04:25,046 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39785 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:25,156 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39785 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:25,267 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39785 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:25,377 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39785 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:25,487 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39785 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:25,596 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39785 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:25,599 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53404 deadline: 1731729864699 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53404 2024-11-16T04:04:25,599 WARN [TestNettyTlsIPC-1-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:25,600 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53404 deadline: 1731729864699 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53404 2024-11-16T04:04:25,600 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53404 deadline: 1731729864699 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53404 2024-11-16T04:04:25,600 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53404 deadline: 1731729864699 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53404 2024-11-16T04:04:25,600 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53404 deadline: 1731729864699 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53404 2024-11-16T04:04:25,601 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53404 deadline: 1731729864699 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53404 2024-11-16T04:04:25,601 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53404 deadline: 1731729864699 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53404 2024-11-16T04:04:25,601 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53404 deadline: 1731729864699 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53404 2024-11-16T04:04:25,601 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:53404 deadline: 1731729864699 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:53404 2024-11-16T04:04:25,706 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39785 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:25,816 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39785 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:25,880 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51022 deadline: 1731729865037 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51022 2024-11-16T04:04:25,880 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51022 deadline: 1731729865147 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51022 2024-11-16T04:04:25,880 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:04:25,880 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51022 deadline: 1731729865257 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51022 2024-11-16T04:04:25,880 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51022 deadline: 1731729865368 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51022 2024-11-16T04:04:25,880 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51022 deadline: 1731729865478 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51022 2024-11-16T04:04:25,880 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51022 deadline: 1731729865588 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51022 2024-11-16T04:04:25,880 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51022 deadline: 1731729865698 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51022 2024-11-16T04:04:25,881 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:51022 deadline: 1731729865808 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:51022 2024-11-16T04:04:25,926 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39785 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:25,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:25,927 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39785 2024-11-16T04:04:25,940 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=65 (was 66), OpenFileDescriptor=380 (was 380), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6842 (was 6843) 2024-11-16T04:04:25,945 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=65, OpenFileDescriptor=380, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6842 2024-11-16T04:04:25,946 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:25,946 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43963 2024-11-16T04:04:25,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dd8088f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:25,947 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:25,947 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:25,947 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:25,976 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:25,976 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:25,990 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:26,002 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:26,002 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x39b74ec9, L:/127.0.0.1:43963 - R:/127.0.0.1:41306] 2024-11-16T04:04:26,028 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:26,028 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:26,029 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:41306 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:26,029 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:26,030 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:26,030 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:26,030 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:26,035 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66 (was 65) - Thread LEAK? -, OpenFileDescriptor=381 (was 380) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6842 (was 6842) 2024-11-16T04:04:26,041 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66, OpenFileDescriptor=381, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6842 2024-11-16T04:04:26,042 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:26,042 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41007 2024-11-16T04:04:26,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2010120e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:26,043 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:26,043 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:26,044 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:26,073 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:26,074 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:26,087 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:26,099 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:26,099 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xd9a4dd84, L:/127.0.0.1:41007 - R:/127.0.0.1:42712] 2024-11-16T04:04:26,125 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:26,132 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:26,133 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41007 2024-11-16T04:04:26,140 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66 (was 66), OpenFileDescriptor=381 (was 381), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6842 (was 6842) 2024-11-16T04:04:26,145 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66, OpenFileDescriptor=381, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6842 2024-11-16T04:04:26,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55dadeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:26,146 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:26,147 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40835 2024-11-16T04:04:26,147 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:26,147 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:26,147 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:26,172 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:26,173 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:26,186 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:26,198 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:26,198 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x00e9f84c, L:/127.0.0.1:40835 - R:/127.0.0.1:57104] 2024-11-16T04:04:26,225 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:26,225 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:26,225 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:57104 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:26,226 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:26,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:26,226 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40835 2024-11-16T04:04:26,231 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66 (was 66), OpenFileDescriptor=381 (was 381), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6842 (was 6842) 2024-11-16T04:04:26,235 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66, OpenFileDescriptor=381, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6841 2024-11-16T04:04:26,236 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:26,237 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44269 2024-11-16T04:04:26,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@780a39c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:26,237 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:26,237 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:26,237 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:26,265 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:26,265 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:26,279 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:26,292 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:26,292 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xea05a5dd, L:/127.0.0.1:44269 - R:/127.0.0.1:60210] 2024-11-16T04:04:26,319 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:26,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:26,322 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44269 2024-11-16T04:04:26,327 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66 (was 66), OpenFileDescriptor=381 (was 381), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6842 (was 6841) - AvailableMemoryMB LEAK? - 2024-11-16T04:04:26,332 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66, OpenFileDescriptor=381, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6841 2024-11-16T04:04:26,333 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:26,333 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35697 2024-11-16T04:04:26,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f8362f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:26,333 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:26,334 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:26,334 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:26,363 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:26,363 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:26,377 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:26,389 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:26,389 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x513a5638, L:/127.0.0.1:35697 - R:/127.0.0.1:57148] 2024-11-16T04:04:26,417 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:26,417 WARN [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:57148; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:26,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:26,418 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:35697 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:04:26,418 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35697 2024-11-16T04:04:26,423 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66 (was 66), OpenFileDescriptor=381 (was 381), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6841 (was 6841) 2024-11-16T04:04:26,428 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66, OpenFileDescriptor=381, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6841 2024-11-16T04:04:26,429 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:26,429 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40047 2024-11-16T04:04:26,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2724d51c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:26,430 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:26,430 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:26,430 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:26,458 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:26,458 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:26,472 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:26,484 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:26,484 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x338353d3, L:/127.0.0.1:40047 - R:/127.0.0.1:59816] 2024-11-16T04:04:26,511 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:04:26,511 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:04:26,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:26,516 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66 (was 66), OpenFileDescriptor=382 (was 381) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6841 (was 6841) 2024-11-16T04:04:26,521 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66, OpenFileDescriptor=382, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6841 2024-11-16T04:04:26,522 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:26,522 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36771 2024-11-16T04:04:26,523 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f8ac660, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:26,523 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:26,523 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:26,523 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:26,551 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:26,551 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:26,564 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:26,577 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:26,577 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xd98986f5, L:/127.0.0.1:36771 - R:/127.0.0.1:60512] 2024-11-16T04:04:26,602 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:04:26,603 WARN [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:60512 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:60512 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:26,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:26,608 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66 (was 66), OpenFileDescriptor=383 (was 382) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6841 (was 6841) 2024-11-16T04:04:26,613 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66, OpenFileDescriptor=383, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6841 2024-11-16T04:04:26,614 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:26,614 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42211 2024-11-16T04:04:26,614 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fce25d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:26,614 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:26,615 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:26,615 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:26,642 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:26,642 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:26,655 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:26,669 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:26,669 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x7f48e7aa, L:/127.0.0.1:42211 - R:/127.0.0.1:44778] 2024-11-16T04:04:26,694 WARN [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:44778 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:44778 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:26,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:26,695 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42211 2024-11-16T04:04:26,700 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66 (was 66), OpenFileDescriptor=383 (was 383), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6841 (was 6841) 2024-11-16T04:04:26,704 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66, OpenFileDescriptor=383, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6841 2024-11-16T04:04:26,705 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:26,705 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42169 2024-11-16T04:04:26,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dd1df01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:26,705 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:26,706 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:26,706 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:26,734 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:26,734 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:26,749 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:26,761 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:26,761 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xe8304430, L:/127.0.0.1:42169 - R:/127.0.0.1:34130] 2024-11-16T04:04:26,787 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:26,881 WARN [TestNettyTlsIPC-1-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:26,889 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:26,889 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:26,890 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:26,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:26,899 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66 (was 66), OpenFileDescriptor=384 (was 383) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6841 (was 6841) 2024-11-16T04:04:26,904 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66, OpenFileDescriptor=384, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6841 2024-11-16T04:04:26,905 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:26,905 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34811 2024-11-16T04:04:26,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de0ea52, compressor=org.apache.hadoop.io.compress.GzipCodec@ecd8147, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:26,906 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:26,906 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:26,906 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:26,934 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:26,934 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:26,947 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:26,959 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:26,959 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xe5a05016, L:/127.0.0.1:34811 - R:/127.0.0.1:51180] 2024-11-16T04:04:26,986 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:26,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:26,988 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34811 2024-11-16T04:04:26,992 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66 (was 66), OpenFileDescriptor=384 (was 384), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6841 (was 6841) 2024-11-16T04:04:26,997 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66, OpenFileDescriptor=384, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6841 2024-11-16T04:04:26,998 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:26,998 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42773 2024-11-16T04:04:26,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e0ce04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:26,998 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:26,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:26,999 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:27,027 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:27,027 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:27,041 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:27,053 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:27,053 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xa59d78e7, L:/127.0.0.1:42773 - R:/127.0.0.1:43178] 2024-11-16T04:04:27,079 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:27,080 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:27,081 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42773 2024-11-16T04:04:27,086 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66 (was 66), OpenFileDescriptor=384 (was 384), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6841 (was 6841) 2024-11-16T04:04:27,091 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66, OpenFileDescriptor=384, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6841 2024-11-16T04:04:27,092 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:27,093 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41869 2024-11-16T04:04:27,093 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c077997, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:27,093 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:27,093 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:27,093 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:27,118 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:27,118 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:27,131 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:27,144 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:27,144 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0x542721e4, L:/127.0.0.1:41869 - R:/127.0.0.1:33144] 2024-11-16T04:04:27,171 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:27,172 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:27,172 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:33144 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:27,172 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:27,173 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:27,173 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41869 2024-11-16T04:04:27,178 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66 (was 66), OpenFileDescriptor=384 (was 384), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6841 (was 6841) 2024-11-16T04:04:27,183 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66, OpenFileDescriptor=384, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6841 2024-11-16T04:04:27,184 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:27,185 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42479 2024-11-16T04:04:27,185 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:27,185 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:27,185 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:27,185 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:27,210 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:27,210 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:27,224 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:27,236 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:27,236 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xbbc39a1d, L:/127.0.0.1:42479 - R:/127.0.0.1:55278] 2024-11-16T04:04:27,264 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:27,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:27,265 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42479 2024-11-16T04:04:27,270 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66 (was 66), OpenFileDescriptor=384 (was 384), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6841 (was 6841) 2024-11-16T04:04:27,276 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66, OpenFileDescriptor=384, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6841 2024-11-16T04:04:27,277 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:27,277 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37735 2024-11-16T04:04:27,278 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7349e6c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:27,278 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:27,278 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:27,278 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:27,309 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:27,309 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:27,322 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:27,335 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:27,335 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xc1438a51, L:/127.0.0.1:37735 - R:/127.0.0.1:51320] 2024-11-16T04:04:27,363 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:27,363 WARN [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:27,363 INFO [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-eaba52f2108eab9fe11e9ec4c253159e-a89f4b259ee84fd1-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:04:27,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:27,363 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:27,363 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37735 2024-11-16T04:04:27,368 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66 (was 66), OpenFileDescriptor=384 (was 384), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6841 (was 6841) 2024-11-16T04:04:27,374 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=66, OpenFileDescriptor=384, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6841 2024-11-16T04:04:27,375 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:27,375 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40699 2024-11-16T04:04:27,376 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c9458aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:27,376 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:27,376 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:27,376 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:27,407 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:27,407 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:27,420 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:27,433 DEBUG [TestNettyTlsIPC-1-2 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:27,433 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.NettyRpcServer(426): SSL handler added for channel: [id: 0xaae4d06d, L:/127.0.0.1:40699 - R:/127.0.0.1:59338] 2024-11-16T04:04:27,461 INFO [TestNettyTlsIPC-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:27,517 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40699 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=140ms, rpcTimeout=100ms 2024-11-16T04:04:27,517 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40699 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=140ms, rpcTimeout=100ms 2024-11-16T04:04:27,517 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40699 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=140ms, rpcTimeout=100ms 2024-11-16T04:04:27,517 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40699 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=140ms, rpcTimeout=100ms 2024-11-16T04:04:27,517 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40699 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=140ms, rpcTimeout=100ms 2024-11-16T04:04:27,517 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40699 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=140ms, rpcTimeout=100ms 2024-11-16T04:04:27,517 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40699 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=140ms, rpcTimeout=100ms 2024-11-16T04:04:27,517 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40699 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=140ms, rpcTimeout=100ms 2024-11-16T04:04:27,517 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40699 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=140ms, rpcTimeout=100ms 2024-11-16T04:04:27,517 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40699 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=140ms, rpcTimeout=100ms 2024-11-16T04:04:27,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:27,517 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40699 2024-11-16T04:04:27,523 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[21: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=false, clientTlsEnabled=true] Thread=67 (was 66) - Thread LEAK? -, OpenFileDescriptor=384 (was 384), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6841 (was 6841) 2024-11-16T04:04:27,530 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=67, OpenFileDescriptor=384, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6841 2024-11-16T04:04:27,531 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:27,531 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37447 2024-11-16T04:04:27,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b3b4bbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:27,531 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:27,531 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:27,532 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:27,532 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:37447 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:04:27,532 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37447 2024-11-16T04:04:27,537 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=67 (was 67), OpenFileDescriptor=384 (was 384), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6841 (was 6841) 2024-11-16T04:04:27,542 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=67, OpenFileDescriptor=384, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6841 2024-11-16T04:04:27,543 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:27,543 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40741 2024-11-16T04:04:27,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34f154f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:27,543 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:27,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:27,544 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:27,568 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:27,568 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:27,581 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:27,594 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:27,594 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x9d695427, L:/127.0.0.1:40741 - R:/127.0.0.1:42978] 2024-11-16T04:04:27,621 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:27,676 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40741 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=133ms, rpcTimeout=100ms 2024-11-16T04:04:27,786 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40741 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:27,896 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40741 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:28,006 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40741 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:28,116 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40741 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:28,227 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40741 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:28,337 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40741 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:28,447 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40741 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:28,462 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:59338 deadline: 1731729867561 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:59338 2024-11-16T04:04:28,462 WARN [TestNettyTlsIPC-1-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:28,462 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:59338 deadline: 1731729867562 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:59338 2024-11-16T04:04:28,463 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:59338 deadline: 1731729867562 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:59338 2024-11-16T04:04:28,463 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:59338 deadline: 1731729867562 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:59338 2024-11-16T04:04:28,463 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:59338 deadline: 1731729867562 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:59338 2024-11-16T04:04:28,463 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:59338 deadline: 1731729867562 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:59338 2024-11-16T04:04:28,463 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:59338 deadline: 1731729867562 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:59338 2024-11-16T04:04:28,464 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:59338 deadline: 1731729867562 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:59338 2024-11-16T04:04:28,464 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:59338 deadline: 1731729867562 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:59338 2024-11-16T04:04:28,556 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40741 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:28,622 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42978 deadline: 1731729867777 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42978 2024-11-16T04:04:28,623 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:04:28,623 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42978 deadline: 1731729867887 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42978 2024-11-16T04:04:28,623 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42978 deadline: 1731729867998 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42978 2024-11-16T04:04:28,623 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42978 deadline: 1731729868107 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42978 2024-11-16T04:04:28,624 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42978 deadline: 1731729868217 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42978 2024-11-16T04:04:28,624 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42978 deadline: 1731729868328 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42978 2024-11-16T04:04:28,624 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42978 deadline: 1731729868438 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42978 2024-11-16T04:04:28,624 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:42978 deadline: 1731729868548 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:42978 2024-11-16T04:04:28,666 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:40741 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:28,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:28,666 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40741 2024-11-16T04:04:28,671 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=67 (was 67), OpenFileDescriptor=384 (was 384), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6841 (was 6841) 2024-11-16T04:04:28,676 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=67, OpenFileDescriptor=384, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6841 2024-11-16T04:04:28,677 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:28,678 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34279 2024-11-16T04:04:28,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e1c6f03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:28,678 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:28,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:28,678 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:28,703 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:28,703 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:28,716 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:28,729 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:28,729 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xc409bd32, L:/127.0.0.1:34279 - R:/127.0.0.1:40954] 2024-11-16T04:04:28,756 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:28,756 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:28,756 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:40954 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:28,757 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:28,758 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:28,758 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:28,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:28,763 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68 (was 67) - Thread LEAK? -, OpenFileDescriptor=385 (was 384) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6841 (was 6841) 2024-11-16T04:04:28,769 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68, OpenFileDescriptor=385, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6841 2024-11-16T04:04:28,770 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:28,770 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44593 2024-11-16T04:04:28,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c405c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:28,770 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:28,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:28,771 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:28,799 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:28,799 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:28,813 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:28,825 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:28,826 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x801834ee, L:/127.0.0.1:44593 - R:/127.0.0.1:40722] 2024-11-16T04:04:28,852 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:28,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:28,859 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44593 2024-11-16T04:04:28,864 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68 (was 68), OpenFileDescriptor=385 (was 385), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6841 (was 6841) 2024-11-16T04:04:28,869 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68, OpenFileDescriptor=385, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6841 2024-11-16T04:04:28,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5193c922, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:28,870 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:28,871 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33101 2024-11-16T04:04:28,871 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:28,871 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:28,871 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:28,895 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:28,896 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:28,908 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:28,921 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:28,921 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xb5b6e5dc, L:/127.0.0.1:33101 - R:/127.0.0.1:57698] 2024-11-16T04:04:28,948 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:28,948 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:28,948 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:57698 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:28,949 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:28,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:28,949 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33101 2024-11-16T04:04:28,955 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68 (was 68), OpenFileDescriptor=385 (was 385), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6840 (was 6841) 2024-11-16T04:04:28,960 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68, OpenFileDescriptor=385, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6840 2024-11-16T04:04:28,961 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:28,961 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37319 2024-11-16T04:04:28,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77533eab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:28,961 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:28,962 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:28,962 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:28,987 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:28,987 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:29,000 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:29,012 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:29,013 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x353a2d6f, L:/127.0.0.1:37319 - R:/127.0.0.1:36224] 2024-11-16T04:04:29,039 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:29,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:29,042 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37319 2024-11-16T04:04:29,047 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68 (was 68), OpenFileDescriptor=385 (was 385), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6840 (was 6840) 2024-11-16T04:04:29,053 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68, OpenFileDescriptor=385, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6840 2024-11-16T04:04:29,054 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:29,054 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44493 2024-11-16T04:04:29,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10c901af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:29,054 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:29,055 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:29,055 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:29,080 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:29,080 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:29,093 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:29,105 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:29,105 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x15c5f422, L:/127.0.0.1:44493 - R:/127.0.0.1:46486] 2024-11-16T04:04:29,132 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:29,133 WARN [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:46486; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:29,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:29,133 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:44493 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:04:29,133 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44493 2024-11-16T04:04:29,139 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68 (was 68), OpenFileDescriptor=385 (was 385), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6840 (was 6840) 2024-11-16T04:04:29,144 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68, OpenFileDescriptor=385, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6840 2024-11-16T04:04:29,145 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:29,145 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45513 2024-11-16T04:04:29,145 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a14c5e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:29,145 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:29,146 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:29,146 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:29,171 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:29,171 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:29,184 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:29,197 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:29,198 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xb07c181f, L:/127.0.0.1:45513 - R:/127.0.0.1:58916] 2024-11-16T04:04:29,225 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:04:29,225 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:04:29,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:29,230 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68 (was 68), OpenFileDescriptor=386 (was 385) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6840 (was 6840) 2024-11-16T04:04:29,235 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68, OpenFileDescriptor=386, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6839 2024-11-16T04:04:29,236 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:29,236 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33527 2024-11-16T04:04:29,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5911048a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:29,236 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:29,237 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:29,237 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:29,264 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:29,264 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:29,279 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:29,291 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:29,291 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x6ce2670d, L:/127.0.0.1:33527 - R:/127.0.0.1:56186] 2024-11-16T04:04:29,317 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:04:29,317 WARN [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:56186 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:56186 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:29,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:29,323 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68 (was 68), OpenFileDescriptor=387 (was 386) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6839 (was 6839) 2024-11-16T04:04:29,329 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68, OpenFileDescriptor=387, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6839 2024-11-16T04:04:29,330 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:29,330 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42817 2024-11-16T04:04:29,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3467e0c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:29,331 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:29,331 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:29,331 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:29,357 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:29,358 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:29,370 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:29,383 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:29,383 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x3a229fe4, L:/127.0.0.1:42817 - R:/127.0.0.1:51704] 2024-11-16T04:04:29,410 WARN [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:51704 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:51704 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1495) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:29,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:29,410 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42817 2024-11-16T04:04:29,417 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68 (was 68), OpenFileDescriptor=387 (was 387), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6839 (was 6839) 2024-11-16T04:04:29,423 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68, OpenFileDescriptor=387, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6839 2024-11-16T04:04:29,424 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:29,424 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43883 2024-11-16T04:04:29,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@645e9844, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:29,424 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:29,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:29,425 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:29,450 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:29,450 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:29,463 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:29,475 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:29,475 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xccda3c4e, L:/127.0.0.1:43883 - R:/127.0.0.1:45170] 2024-11-16T04:04:29,502 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:45170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:29,604 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:29,605 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:29,605 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:29,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:29,617 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=69 (was 68) - Thread LEAK? -, OpenFileDescriptor=388 (was 387) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6839 (was 6839) 2024-11-16T04:04:29,622 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=69, OpenFileDescriptor=388, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6839 2024-11-16T04:04:29,623 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:29,624 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:32975 2024-11-16T04:04:29,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a3dfc67, compressor=org.apache.hadoop.io.compress.GzipCodec@e18b08a, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:29,624 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:29,624 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:29,624 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:29,625 WARN [TestNettyTlsIPC-1-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:29,651 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:29,651 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:29,664 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:29,676 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:29,677 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xe74cedbb, L:/127.0.0.1:32975 - R:/127.0.0.1:51040] 2024-11-16T04:04:29,703 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:29,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:29,705 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:32975 2024-11-16T04:04:29,710 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68 (was 69), OpenFileDescriptor=388 (was 388), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6839 (was 6839) 2024-11-16T04:04:29,716 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68, OpenFileDescriptor=388, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6839 2024-11-16T04:04:29,717 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:29,717 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33515 2024-11-16T04:04:29,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2167d54d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:29,717 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:29,718 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:29,718 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:29,747 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:29,747 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:29,760 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:29,772 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:29,772 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x9dd91f69, L:/127.0.0.1:33515 - R:/127.0.0.1:47152] 2024-11-16T04:04:29,799 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:29,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:29,800 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33515 2024-11-16T04:04:29,805 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68 (was 68), OpenFileDescriptor=388 (was 388), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6839 (was 6839) 2024-11-16T04:04:29,810 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68, OpenFileDescriptor=388, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6839 2024-11-16T04:04:29,811 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:29,811 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45261 2024-11-16T04:04:29,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@326320af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:29,811 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:29,812 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:29,812 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:29,839 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:29,840 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:29,853 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:29,865 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:29,865 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xdf119850, L:/127.0.0.1:45261 - R:/127.0.0.1:53452] 2024-11-16T04:04:29,892 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:29,893 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:29,893 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:53452 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:29,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:29,893 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:29,893 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45261 2024-11-16T04:04:29,899 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68 (was 68), OpenFileDescriptor=388 (was 388), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6838 (was 6839) 2024-11-16T04:04:29,904 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68, OpenFileDescriptor=388, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6838 2024-11-16T04:04:29,905 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:29,905 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44111 2024-11-16T04:04:29,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:29,906 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:29,906 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:29,906 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:29,931 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:29,931 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:29,944 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:29,956 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:29,956 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x9ee601e4, L:/127.0.0.1:44111 - R:/127.0.0.1:48834] 2024-11-16T04:04:29,983 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:29,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:29,984 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44111 2024-11-16T04:04:29,989 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68 (was 68), OpenFileDescriptor=388 (was 388), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6838 (was 6838) 2024-11-16T04:04:29,995 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68, OpenFileDescriptor=388, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6838 2024-11-16T04:04:29,996 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:29,996 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38423 2024-11-16T04:04:29,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e94d564, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:29,996 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:29,996 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:29,997 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:30,021 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:30,022 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:30,035 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:30,047 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:30,047 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x3eb09012, L:/127.0.0.1:38423 - R:/127.0.0.1:38450] 2024-11-16T04:04:30,074 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:30,074 WARN [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:30,074 INFO [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-d76231281290a154c60855b73d60680d-b4eb8baecf83187e-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:04:30,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:30,075 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:30,075 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38423 2024-11-16T04:04:30,080 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68 (was 68), OpenFileDescriptor=388 (was 388), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6838 (was 6838) 2024-11-16T04:04:30,085 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=68, OpenFileDescriptor=388, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6837 2024-11-16T04:04:30,086 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:30,086 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36227 2024-11-16T04:04:30,087 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b9979ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:30,087 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:30,087 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:30,087 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:30,112 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:30,112 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcConnection$3(382): SSL handler added with handshake timeout 5000 ms 2024-11-16T04:04:30,125 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:30,137 DEBUG [TestNettyTlsIPC-1-1 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:30,137 DEBUG [TestNettyTlsIPC-1-1 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x578a4568, L:/127.0.0.1:36227 - R:/127.0.0.1:40180] 2024-11-16T04:04:30,174 INFO [TestNettyTlsIPC-1-1 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:30,217 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:04:30,217 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:04:30,217 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:04:30,217 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:04:30,217 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=130ms, rpcTimeout=100ms 2024-11-16T04:04:30,217 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=130ms, rpcTimeout=100ms 2024-11-16T04:04:30,217 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:04:30,217 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:04:30,217 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:04:30,217 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:36227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=129ms, rpcTimeout=100ms 2024-11-16T04:04:30,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:30,217 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36227 2024-11-16T04:04:30,223 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[22: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=true] Thread=69 (was 68) - Thread LEAK? -, OpenFileDescriptor=388 (was 388), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6836 (was 6837) 2024-11-16T04:04:30,228 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=69, OpenFileDescriptor=388, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6836 2024-11-16T04:04:30,229 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:30,229 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36257 2024-11-16T04:04:30,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e7bded5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:30,229 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:30,229 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:30,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:30,230 INFO [Time-limited test {}] ipc.AbstractTestIPC(224): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.io.IOException: Call to address=localhost:36257 failed on local exception: java.io.IOException: java.lang.RuntimeException: Injected fault 2024-11-16T04:04:30,230 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36257 2024-11-16T04:04:30,234 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRTEDuringConnectionSetup[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=69 (was 69), OpenFileDescriptor=388 (was 388), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6836 (was 6836) 2024-11-16T04:04:30,239 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTimeout[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=69, OpenFileDescriptor=388, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6835 2024-11-16T04:04:30,240 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:30,240 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39419 2024-11-16T04:04:30,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b2a704, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:30,240 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:30,240 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:30,257 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:30,269 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:30,269 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xbbf65bf7, L:/127.0.0.1:39419 - R:/127.0.0.1:56442] 2024-11-16T04:04:30,269 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:30,346 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=106ms, rpcTimeout=100ms 2024-11-16T04:04:30,456 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:30,566 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:30,676 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:30,787 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=110ms, rpcTimeout=100ms 2024-11-16T04:04:30,897 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:31,006 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:31,116 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:31,175 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:40180 deadline: 1731729870275 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:40180 2024-11-16T04:04:31,175 WARN [TestNettyTlsIPC-1-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:31,176 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:40180 deadline: 1731729870275 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:40180 2024-11-16T04:04:31,176 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:40180 deadline: 1731729870275 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:40180 2024-11-16T04:04:31,176 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:40180 deadline: 1731729870275 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:40180 2024-11-16T04:04:31,176 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:40180 deadline: 1731729870275 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:40180 2024-11-16T04:04:31,177 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:40180 deadline: 1731729870275 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:40180 2024-11-16T04:04:31,177 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:40180 deadline: 1731729870275 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:40180 2024-11-16T04:04:31,177 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:40180 deadline: 1731729870275 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:40180 2024-11-16T04:04:31,177 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:40180 deadline: 1731729870275 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:40180 2024-11-16T04:04:31,226 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:31,271 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:56442 deadline: 1731729870448 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:56442 2024-11-16T04:04:31,271 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.RpcConnection(450): Unknown callId: 0, skipping over this response of 1 bytes 2024-11-16T04:04:31,271 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:56442 deadline: 1731729870557 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:56442 2024-11-16T04:04:31,271 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:56442 deadline: 1731729870667 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:56442 2024-11-16T04:04:31,272 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:56442 deadline: 1731729870777 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:56442 2024-11-16T04:04:31,272 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:56442 deadline: 1731729870888 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:56442 2024-11-16T04:04:31,272 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:56442 deadline: 1731729870998 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:56442 2024-11-16T04:04:31,272 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:56442 deadline: 1731729871108 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:56442 2024-11-16T04:04:31,272 WARN [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(100): Dropping timed out call: callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:56442 deadline: 1731729871217 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:56442 2024-11-16T04:04:31,336 INFO [Time-limited test {}] ipc.AbstractTestIPC(349): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:39419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:31,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:31,336 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39419 2024-11-16T04:04:31,344 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTimeout[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=69 (was 69), OpenFileDescriptor=388 (was 388), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6834 (was 6835) 2024-11-16T04:04:31,351 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingErrorIpc[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=69, OpenFileDescriptor=388, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6834 2024-11-16T04:04:31,352 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:31,353 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45523 2024-11-16T04:04:31,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ad24f42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:31,353 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:31,353 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:31,370 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:31,382 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:31,382 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x5db132cc, L:/127.0.0.1:45523 - R:/127.0.0.1:43566] 2024-11-16T04:04:31,382 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:31,383 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:31,383 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:43566 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:31,384 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:31,384 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:31,384 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:31,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:31,389 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingErrorIpc[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=70 (was 69) - Thread LEAK? -, OpenFileDescriptor=389 (was 388) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6834 (was 6834) 2024-11-16T04:04:31,394 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcScheduler[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=70, OpenFileDescriptor=389, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6834 2024-11-16T04:04:31,395 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:31,395 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42407 2024-11-16T04:04:31,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cb51191, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:31,396 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler$MockitoMock$yonKblHL as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:31,396 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:31,414 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:31,426 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:31,426 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xb3b43c2f, L:/127.0.0.1:42407 - R:/127.0.0.1:41890] 2024-11-16T04:04:31,426 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:31,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:31,432 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42407 2024-11-16T04:04:31,438 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcScheduler[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=70 (was 70), OpenFileDescriptor=389 (was 389), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6834 (was 6834) 2024-11-16T04:04:31,444 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncRemoteError[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=70, OpenFileDescriptor=389, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6834 2024-11-16T04:04:31,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c891a09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:31,446 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:31,446 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36261 2024-11-16T04:04:31,446 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:31,446 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:31,459 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:31,472 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:31,472 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x50fb05be, L:/127.0.0.1:36261 - R:/127.0.0.1:42778] 2024-11-16T04:04:31,472 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:31,473 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:31,473 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:42778 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:31,473 INFO [Time-limited test {}] ipc.AbstractTestIPC(466): Caught expected exception: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:31,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:31,473 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36261 2024-11-16T04:04:31,479 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncRemoteError[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=70 (was 70), OpenFileDescriptor=389 (was 389), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6834 (was 6834) 2024-11-16T04:04:31,484 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncEcho[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=70, OpenFileDescriptor=389, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6834 2024-11-16T04:04:31,485 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:31,486 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35927 2024-11-16T04:04:31,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15f31697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:31,486 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:31,486 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:31,499 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:31,511 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:31,511 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x5ede6216, L:/127.0.0.1:35927 - R:/127.0.0.1:55298] 2024-11-16T04:04:31,512 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:31,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:31,514 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35927 2024-11-16T04:04:31,519 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncEcho[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=70 (was 70), OpenFileDescriptor=389 (was 389), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6834 (was 6834) 2024-11-16T04:04:31,524 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=70, OpenFileDescriptor=389, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6834 2024-11-16T04:04:31,526 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:31,526 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35883 2024-11-16T04:04:31,526 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c12b160, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:31,526 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:31,526 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:31,545 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:31,557 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:31,557 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x34b4f186, L:/127.0.0.1:35883 - R:/127.0.0.1:35846] 2024-11-16T04:04:31,557 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:31,558 WARN [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /127.0.0.1:35846; caught unexpected downstream exception. org.apache.hadoop.hbase.DoNotRetryIOException: Failing for test at org.apache.hadoop.hbase.ipc.FailingNettyRpcServer$FailingConnection.processRequest(FailingNettyRpcServer.java:47) ~[test-classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.callHandlerRemoved(AbstractChannelHandlerContext.java:1138) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:586) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:426) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.remove(DefaultChannelPipeline.java:372) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:95) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:31,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:31,558 INFO [Time-limited test {}] ipc.AbstractTestIPC(414): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=localhost:35883 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-16T04:04:31,558 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35883 2024-11-16T04:04:31,563 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testConnectionCloseWithOutstandingRPCs[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=70 (was 70), OpenFileDescriptor=389 (was 389), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6834 (was 6834) 2024-11-16T04:04:31,568 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistry[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=70, OpenFileDescriptor=389, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6834 2024-11-16T04:04:31,569 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:31,569 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44371 2024-11-16T04:04:31,569 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@759ad4f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:31,569 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:31,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:31,586 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:31,598 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:31,598 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x07891930, L:/127.0.0.1:44371 - R:/127.0.0.1:34028] 2024-11-16T04:04:31,598 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'test_cluster_id' 2024-11-16T04:04:31,598 DEBUG [TestNettyTlsIPC-1-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:04:31,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:31,603 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistry[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=70 (was 70), OpenFileDescriptor=390 (was 389) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6834 (was 6834) 2024-11-16T04:04:31,608 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=70, OpenFileDescriptor=390, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6834 2024-11-16T04:04:31,609 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:31,609 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34681 2024-11-16T04:04:31,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36f77858, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:31,609 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:31,610 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:04:31,626 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:31,639 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:31,639 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x05d08d4b, L:/127.0.0.1:34681 - R:/127.0.0.1:49516] 2024-11-16T04:04:31,639 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'null' 2024-11-16T04:04:31,639 WARN [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(695): Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:49516 org.apache.hadoop.hbase.ipc.FatalConnectionException: Expected HEADER=HBas but received HEADER=Regi from 127.0.0.1:49516 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.doBadPreambleHandling(ServerRpcConnection.java:691) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:766) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:31,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:31,644 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testGetConnectionRegistryError[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=70 (was 70), OpenFileDescriptor=391 (was 390) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6834 (was 6834) 2024-11-16T04:04:31,649 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testBadPreambleHeader[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=70, OpenFileDescriptor=391, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6834 2024-11-16T04:04:31,650 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:31,650 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38183 2024-11-16T04:04:31,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18cc3c23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:31,650 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:31,651 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:31,667 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:31,679 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:31,680 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x78ad9e14, L:/127.0.0.1:38183 - R:/127.0.0.1:55136] 2024-11-16T04:04:31,680 WARN [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(695): serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:55136 org.apache.hadoop.hbase.ipc.BadAuthException: serverVersion=0, clientVersion=0, authMethod=-10, authName=unknown from 127.0.0.1:55136 at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processPreamble(ServerRpcConnection.java:785) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:68) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerPreambleHandler.channelRead0(NettyRpcServerPreambleHandler.java:36) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.handlerRemoved(ByteToMessageDecoder.java:266) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:537) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:31,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:31,680 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38183 2024-11-16T04:04:31,685 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testBadPreambleHeader[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=70 (was 70), OpenFileDescriptor=391 (was 391), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6834 (was 6834) 2024-11-16T04:04:31,690 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testTracingSuccessIpc[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=70, OpenFileDescriptor=391, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6834 2024-11-16T04:04:31,691 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:31,691 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45335 2024-11-16T04:04:31,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75bed7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:31,691 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:31,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:31,708 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:31,720 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:31,720 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x8400387a, L:/127.0.0.1:45335 - R:/127.0.0.1:40846] 2024-11-16T04:04:31,720 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:31,822 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:31,822 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:31,822 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T04:04:31,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:31,833 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testTracingSuccessIpc[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=71 (was 70) - Thread LEAK? -, OpenFileDescriptor=392 (was 391) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6834 (was 6834) 2024-11-16T04:04:31,841 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testCompressCellBlock[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=71, OpenFileDescriptor=392, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6834 2024-11-16T04:04:31,842 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:31,842 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39493 2024-11-16T04:04:31,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d9af4f9, compressor=org.apache.hadoop.io.compress.GzipCodec@67770ca9, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:31,843 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:31,843 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:31,859 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:31,871 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:31,871 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x1bcbeb08, L:/127.0.0.1:39493 - R:/127.0.0.1:34982] 2024-11-16T04:04:31,871 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:31,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:31,872 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39493 2024-11-16T04:04:31,878 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testCompressCellBlock[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=71 (was 71), OpenFileDescriptor=392 (was 392), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6834 (was 6834) 2024-11-16T04:04:31,882 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=71, OpenFileDescriptor=392, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6834 2024-11-16T04:04:31,883 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:31,883 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40547 2024-11-16T04:04:31,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b008f12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:31,884 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:31,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:31,900 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:31,912 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:31,912 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xb85607e2, L:/127.0.0.1:40547 - R:/127.0.0.1:59358] 2024-11-16T04:04:31,912 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:31,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:31,913 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40547 2024-11-16T04:04:31,918 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcServerForNotNullRemoteAddressInCallObject[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=71 (was 71), OpenFileDescriptor=392 (was 392), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6833 (was 6834) 2024-11-16T04:04:31,923 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRemoteError[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=71, OpenFileDescriptor=392, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6833 2024-11-16T04:04:31,923 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:31,924 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36977 2024-11-16T04:04:31,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c58be44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:31,924 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:31,924 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:31,939 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:31,951 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:31,952 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x5ed50b1b, L:/127.0.0.1:36977 - R:/127.0.0.1:42580] 2024-11-16T04:04:31,952 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:31,952 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:04:31,953 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(138): callId: 0 service: TestProtobufRpcProto methodName: error size: 91 connection: 127.0.0.1:42580 deadline: 9223372036854775807, exception=org.apache.hadoop.hbase.DoNotRetryIOException: server error! 2024-11-16T04:04:31,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:31,953 INFO [Time-limited test {}] ipc.AbstractTestIPC(319): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException): org.apache.hadoop.hbase.DoNotRetryIOException: server error! at org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.error(TestProtobufRpcServiceImpl.java:106) at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$2.callBlockingMethod(TestRpcServiceProtos.java:153) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:112) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:04:31,953 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36977 2024-11-16T04:04:31,958 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRemoteError[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=71 (was 71), OpenFileDescriptor=392 (was 392), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6833 (was 6833) 2024-11-16T04:04:31,963 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testNoCodec[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=71, OpenFileDescriptor=392, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6833 2024-11-16T04:04:31,964 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:31,964 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37459 2024-11-16T04:04:31,964 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=null, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:31,964 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:31,965 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:31,982 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:31,994 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:31,994 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x2ac341e3, L:/127.0.0.1:37459 - R:/127.0.0.1:50480] 2024-11-16T04:04:31,994 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:31,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:31,995 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37459 2024-11-16T04:04:32,000 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testNoCodec[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=71 (was 71), OpenFileDescriptor=392 (was 392), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6833 (was 6833) 2024-11-16T04:04:32,006 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=71, OpenFileDescriptor=392, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6833 2024-11-16T04:04:32,007 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:32,008 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40287 2024-11-16T04:04:32,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66fcbf5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:32,008 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:32,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:32,022 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:32,034 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:32,035 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0x8d4a88f0, L:/127.0.0.1:40287 - R:/127.0.0.1:33264] 2024-11-16T04:04:32,035 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:32,035 WARN [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcFrameDecoder(85): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:32,035 INFO [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcFrameDecoder(125): BigRequest header is = call_id: 0 trace_info { headers { key: "traceparent" value: "00-745fcd35a30dbde869d7af93079776e4-df6a8cc2554c9cde-01" } } method_name: "echo" request_param: true cell_block_meta { length: 36 } priority: 0 timeout: 0 2024-11-16T04:04:32,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:32,036 INFO [Time-limited test {}] ipc.AbstractTestIPC(278): Caught expected exception: org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RequestTooBigException): RPC data length of 1334 received from 127.0.0.1 is greater than max allowed 1000. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended) 2024-11-16T04:04:32,036 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40287 2024-11-16T04:04:32,041 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testRpcMaxRequestSize[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=71 (was 71), OpenFileDescriptor=392 (was 392), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6833 (was 6833) 2024-11-16T04:04:32,046 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: ipc.TestNettyTlsIPC#testAsyncTimeout[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=71, OpenFileDescriptor=392, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=6833 2024-11-16T04:04:32,047 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:04:32,047 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44857 2024-11-16T04:04:32,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4934fb69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:04:32,047 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-16T04:04:32,047 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-16T04:04:32,060 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(313): Using default JDK SSL provider because netty-tcnative is not available 2024-11-16T04:04:32,073 DEBUG [TestNettyTlsIPC-1-3 {}] tls.X509Util(236): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2024-11-16T04:04:32,073 DEBUG [TestNettyTlsIPC-1-3 {}] ipc.NettyRpcServer(406): Dual mode SSL handler added for channel: [id: 0xab850981, L:/127.0.0.1:44857 - R:/127.0.0.1:46164] 2024-11-16T04:04:32,073 INFO [TestNettyTlsIPC-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-16T04:04:32,157 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44857 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=0,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:32,157 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44857 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:04:32,157 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44857 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=2,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:04:32,157 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44857 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:04:32,157 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44857 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=4,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:04:32,157 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44857 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=5,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:04:32,157 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44857 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=6,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:04:32,157 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44857 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=7,methodName=pause], waitTime=108ms, rpcTimeout=100ms 2024-11-16T04:04:32,157 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44857 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:32,157 INFO [Time-limited test {}] ipc.AbstractTestIPC(504): Caught expected exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=localhost:44857 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=pause], waitTime=109ms, rpcTimeout=100ms 2024-11-16T04:04:32,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:04:32,157 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44857 2024-11-16T04:04:32,165 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: ipc.TestNettyTlsIPC#testAsyncTimeout[23: rpcServerImpl=class org.apache.hadoop.hbase.ipc.NettyRpcServer, caKeyType=EC, certKeyType=EC, keyPassword=[C@bcec031, acceptPlainText=true, clientTlsEnabled=false] Thread=72 (was 71) - Thread LEAK? -, OpenFileDescriptor=392 (was 392), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=6833 (was 6833) 2024-11-16T04:04:32,370 WARN [TestNettyTlsIPC-1-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:33,075 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 1 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:46164 deadline: 1731729872174 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:46164 2024-11-16T04:04:33,075 WARN [TestNettyTlsIPC-1-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-16T04:04:33,075 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 2 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:46164 deadline: 1731729872174 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:46164 2024-11-16T04:04:33,075 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 3 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:46164 deadline: 1731729872174 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:46164 2024-11-16T04:04:33,075 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 4 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:46164 deadline: 1731729872174 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:46164 2024-11-16T04:04:33,076 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 5 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:46164 deadline: 1731729872174 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:46164 2024-11-16T04:04:33,076 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 6 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:46164 deadline: 1731729872175 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:46164 2024-11-16T04:04:33,076 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 7 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:46164 deadline: 1731729872175 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:46164 2024-11-16T04:04:33,076 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 8 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:46164 deadline: 1731729872175 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:46164 2024-11-16T04:04:33,076 DEBUG [FifoRpcScheduler.handler-pool-0 {}] ipc.CallRunner(93): FifoRpcScheduler.handler-pool-0: skipped callId: 9 service: TestProtobufRpcProto methodName: pause size: 94 connection: 127.0.0.1:46164 deadline: 1731729872175 param: TODO: class org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos$PauseRequestProto connection: 127.0.0.1:46164