2025-01-12 17:46:26,276 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4f4c4b1a 2025-01-12 17:46:26,288 main DEBUG Took 0.010180 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2025-01-12 17:46:26,288 main DEBUG PluginManager 'Core' found 129 plugins 2025-01-12 17:46:26,288 main DEBUG PluginManager 'Level' found 0 plugins 2025-01-12 17:46:26,289 main DEBUG PluginManager 'Lookup' found 16 plugins 2025-01-12 17:46:26,290 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-12 17:46:26,300 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2025-01-12 17:46:26,310 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-12 17:46:26,311 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-12 17:46:26,312 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-12 17:46:26,312 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-12 17:46:26,312 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-12 17:46:26,313 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-12 17:46:26,313 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-12 17:46:26,314 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-12 17:46:26,314 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-12 17:46:26,314 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-12 17:46:26,315 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-12 17:46:26,315 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-12 17:46:26,316 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-12 17:46:26,316 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-12 17:46:26,316 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-12 17:46:26,317 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-12 17:46:26,317 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-12 17:46:26,317 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-12 17:46:26,318 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-12 17:46:26,318 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-12 17:46:26,318 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-12 17:46:26,318 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-12 17:46:26,319 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-12 17:46:26,319 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-12 17:46:26,319 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-12 17:46:26,320 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2025-01-12 17:46:26,321 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-12 17:46:26,322 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2025-01-12 17:46:26,324 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2025-01-12 17:46:26,324 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2025-01-12 17:46:26,325 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2025-01-12 17:46:26,325 main DEBUG PluginManager 'Converter' found 47 plugins 2025-01-12 17:46:26,336 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2025-01-12 17:46:26,338 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2025-01-12 17:46:26,340 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2025-01-12 17:46:26,341 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2025-01-12 17:46:26,341 main DEBUG createAppenders(={Console}) 2025-01-12 17:46:26,342 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4f4c4b1a initialized 2025-01-12 17:46:26,342 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4f4c4b1a 2025-01-12 17:46:26,342 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4f4c4b1a OK. 2025-01-12 17:46:26,343 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2025-01-12 17:46:26,343 main DEBUG OutputStream closed 2025-01-12 17:46:26,344 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2025-01-12 17:46:26,344 main DEBUG Appender DefaultConsole-1 stopped with status true 2025-01-12 17:46:26,344 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6b695b06 OK 2025-01-12 17:46:26,435 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2025-01-12 17:46:26,438 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2025-01-12 17:46:26,439 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2025-01-12 17:46:26,441 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2025-01-12 17:46:26,441 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2025-01-12 17:46:26,442 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2025-01-12 17:46:26,442 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2025-01-12 17:46:26,443 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2025-01-12 17:46:26,443 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2025-01-12 17:46:26,444 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2025-01-12 17:46:26,444 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2025-01-12 17:46:26,445 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2025-01-12 17:46:26,445 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2025-01-12 17:46:26,445 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2025-01-12 17:46:26,446 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2025-01-12 17:46:26,446 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2025-01-12 17:46:26,447 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2025-01-12 17:46:26,448 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2025-01-12 17:46:26,451 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2025-01-12 17:46:26,452 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@366ac49b) with optional ClassLoader: null 2025-01-12 17:46:26,452 main DEBUG Shutdown hook enabled. Registering a new one. 2025-01-12 17:46:26,453 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@366ac49b] started OK. 2025-01-12T17:46:26,471 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.security.TestNettyTLSIPCFileWatcher timeout: 104 mins 2025-01-12 17:46:26,475 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2025-01-12 17:46:26,475 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2025-01-12T17:46:27,024 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2025-01-12T17:46:28,070 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[0: keyType=RSA, storeFileType=JKS] Thread=12, OpenFileDescriptor=295, MaxFileDescriptor=1048576, SystemLoadAverage=332, ProcessCount=11, AvailableMemoryMB=2872 2025-01-12T17:46:28,991 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2025-01-12T17:46:29,047 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2025-01-12T17:46:29,065 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2025-01-12T17:46:29,079 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2025-01-12T17:46:29,169 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-12T17:46:29,210 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 24888 (auto-detected) 2025-01-12T17:46:29,212 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2025-01-12T17:46:29,238 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:41257 2025-01-12T17:46:29,241 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-12T17:46:29,280 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70cfeee2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:29,285 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2025-01-12T17:46:29,287 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2025-01-12T17:46:29,629 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:29,688 DEBUG [RPCClient-NioEventLoopGroup-3-1 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:29,755 DEBUG [RPCClient-NioEventLoopGroup-3-1 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:29,881 DEBUG [FileChangeWatcher-hbase_test_key7353823712555633187.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key7353823712555633187.jks thread started 2025-01-12T17:46:29,888 DEBUG [FileChangeWatcher-hbase_test_ca7174725042374894894.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca7174725042374894894.jks thread started 2025-01-12T17:46:29,898 DEBUG [RPCClient-NioEventLoopGroup-3-1 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:29,967 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:29,996 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:30,100 DEBUG [FileChangeWatcher-hbase_test_key7353823712555633187.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key7353823712555633187.jks thread started 2025-01-12T17:46:30,111 DEBUG [FileChangeWatcher-hbase_test_ca7174725042374894894.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca7174725042374894894.jks thread started 2025-01-12T17:46:30,107 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x6b4cf447, L:/127.0.0.1:41257 - R:/127.0.0.1:56504] 2025-01-12T17:46:30,294 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56504, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:30,715 INFO [Time-limited test {}] ipc.AbstractRpcClient(478): The server on localhost,41257,0 is dead - stopping the connection localhost:41257/TestProtobufRpcProto/jenkins (auth:SIMPLE) 2025-01-12T17:46:30,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:30,718 DEBUG [RPCClient-NioEventLoopGroup-3-2 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:30,720 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x32588b10, L:/127.0.0.1:41257 - R:/127.0.0.1:56518] 2025-01-12T17:46:30,736 INFO [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56518, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:30,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:30,739 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:41257 2025-01-12T17:46:30,739 DEBUG [FileChangeWatcher-hbase_test_key7353823712555633187.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:30,739 DEBUG [FileChangeWatcher-hbase_test_key7353823712555633187.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:30,744 DEBUG [FileChangeWatcher-hbase_test_key7353823712555633187.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key7353823712555633187.jks thread finished 2025-01-12T17:46:30,744 DEBUG [FileChangeWatcher-hbase_test_key7353823712555633187.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key7353823712555633187.jks thread finished 2025-01-12T17:46:30,739 DEBUG [FileChangeWatcher-hbase_test_ca7174725042374894894.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:30,744 DEBUG [FileChangeWatcher-hbase_test_ca7174725042374894894.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca7174725042374894894.jks thread finished 2025-01-12T17:46:30,739 DEBUG [FileChangeWatcher-hbase_test_ca7174725042374894894.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:30,745 DEBUG [FileChangeWatcher-hbase_test_ca7174725042374894894.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca7174725042374894894.jks thread finished 2025-01-12T17:46:30,752 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[0: keyType=RSA, storeFileType=JKS] Thread=22 (was 12) Potentially hanging thread: FileChangeWatcher-hbase_test_ca7174725042374894894.jks app//org.apache.logging.log4j.core.impl.ThrowableProxy.formatExtendedStackTraceTo(ThrowableProxy.java:377) app//org.apache.logging.log4j.core.pattern.ExtendedThrowablePatternConverter.format(ExtendedThrowablePatternConverter.java:74) app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:384) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:375) app//org.apache.logging.log4j.core.layout.PatternLayout.toSerializable(PatternLayout.java:216) app//org.apache.logging.log4j.core.layout.PatternLayout.toSerializable(PatternLayout.java:59) app//org.apache.logging.log4j.core.layout.AbstractStringLayout.toByteArray(AbstractStringLayout.java:294) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:145) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2017) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1983) app//org.apache.logging.slf4j.Log4jLogger.debug(Log4jLogger.java:139) app//org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:248) app//org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: TestNettyTLSIPCFileWatcher-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestNettyTLSIPCFileWatcher-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestNettyTLSIPCFileWatcher-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=327 (was 295) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=332 (was 332), ProcessCount=11 (was 11), AvailableMemoryMB=2717 (was 2872) 2025-01-12T17:46:30,765 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[0: keyType=RSA, storeFileType=JKS] Thread=21, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=332, ProcessCount=11, AvailableMemoryMB=2717 2025-01-12T17:46:30,767 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-12T17:46:30,767 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:39425 2025-01-12T17:46:30,768 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-12T17:46:30,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f643c99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:30,769 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:30,770 DEBUG [RPCClient-NioEventLoopGroup-3-3 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:30,809 DEBUG [RPCClient-NioEventLoopGroup-3-3 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:30,810 DEBUG [FileChangeWatcher-hbase_test_key7353823712555633187.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key7353823712555633187.jks thread started 2025-01-12T17:46:30,810 DEBUG [FileChangeWatcher-hbase_test_ca7174725042374894894.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca7174725042374894894.jks thread started 2025-01-12T17:46:30,810 DEBUG [RPCClient-NioEventLoopGroup-3-3 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:30,827 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:30,843 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:30,844 DEBUG [FileChangeWatcher-hbase_test_key7353823712555633187.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key7353823712555633187.jks thread started 2025-01-12T17:46:30,844 DEBUG [FileChangeWatcher-hbase_test_ca7174725042374894894.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca7174725042374894894.jks thread started 2025-01-12T17:46:30,844 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xe619be0e, L:/127.0.0.1:39425 - R:/127.0.0.1:52424] 2025-01-12T17:46:30,889 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:52424, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:30,891 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:30,892 DEBUG [FileChangeWatcher-hbase_test_key7353823712555633187.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:30,892 DEBUG [FileChangeWatcher-hbase_test_ca7174725042374894894.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:30,893 DEBUG [FileChangeWatcher-hbase_test_key7353823712555633187.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key7353823712555633187.jks thread finished 2025-01-12T17:46:30,893 DEBUG [FileChangeWatcher-hbase_test_ca7174725042374894894.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca7174725042374894894.jks thread finished 2025-01-12T17:46:31,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b96e8bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:31,381 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:31,382 DEBUG [RPCClient-NioEventLoopGroup-3-4 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:31,416 DEBUG [RPCClient-NioEventLoopGroup-3-4 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:31,417 DEBUG [FileChangeWatcher-hbase_test_key7353823712555633187.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key7353823712555633187.jks thread started 2025-01-12T17:46:31,417 DEBUG [FileChangeWatcher-hbase_test_ca7174725042374894894.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca7174725042374894894.jks thread started 2025-01-12T17:46:31,417 DEBUG [RPCClient-NioEventLoopGroup-3-4 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:31,419 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x364cca79, L:/127.0.0.1:39425 - R:/127.0.0.1:52430] 2025-01-12T17:46:31,448 WARN [RPCClient-NioEventLoopGroup-3-4 {}] ipc.NettyRpcConnection$2(410): Exception encountered while connecting to the server localhost:39425 javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more 2025-01-12T17:46:31,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:31,451 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:39425 2025-01-12T17:46:31,451 DEBUG [FileChangeWatcher-hbase_test_key7353823712555633187.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:31,451 DEBUG [FileChangeWatcher-hbase_test_key7353823712555633187.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key7353823712555633187.jks thread finished 2025-01-12T17:46:31,451 DEBUG [FileChangeWatcher-hbase_test_ca7174725042374894894.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:31,451 DEBUG [FileChangeWatcher-hbase_test_key7353823712555633187.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:31,451 DEBUG [FileChangeWatcher-hbase_test_ca7174725042374894894.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca7174725042374894894.jks thread finished 2025-01-12T17:46:31,452 DEBUG [FileChangeWatcher-hbase_test_key7353823712555633187.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key7353823712555633187.jks thread finished 2025-01-12T17:46:31,451 DEBUG [RPCClient-NioEventLoopGroup-3-4 {}] ipc.FailedServers(52): Added failed server with address localhost:39425 to list caused by javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed 2025-01-12T17:46:31,451 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(470): Could not get peer certificate on TLS connection from /127.0.0.1:52430, although one is required javax.net.ssl.SSLPeerUnverifiedException: peer not authenticated at sun.security.ssl.SSLSessionImpl.getPeerCertificates(SSLSessionImpl.java:1034) ~[?:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.sslHandshakeCompleteHandler(NettyRpcServer.java:456) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.lambda$initSSL$0(NettyRpcServer.java:446) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.handleUnwrapThrowable(SslHandler.java:1381) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1363) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2025-01-12T17:46:31,452 DEBUG [FileChangeWatcher-hbase_test_ca7174725042374894894.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:31,452 DEBUG [FileChangeWatcher-hbase_test_ca7174725042374894894.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca7174725042374894894.jks thread finished 2025-01-12T17:46:31,453 WARN [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServerPreambleHandler(100): Connection /127.0.0.1:52430; caught unexpected downstream exception. org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.Alert.createSSLException(Alert.java:117) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:365) ~[?:?] at sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:293) ~[?:?] at sun.security.ssl.TransportContext.dispatch(TransportContext.java:204) ~[?:?] at sun.security.ssl.SSLTransport.decode(SSLTransport.java:172) ~[?:?] at sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:736) ~[?:?] at sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:691) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:506) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:482) ~[?:?] at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:679) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:308) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1465) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 15 more 2025-01-12T17:46:31,452 DEBUG [RPCClient-NioEventLoopGroup-3-4 {}] ipc.BufferCallBeforeInitHandler(140): got ssl exception, which should have already been proceeded, log it here to prevent it being passed to netty's TailContext and trigger a confusing WARN message org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more 2025-01-12T17:46:31,467 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[0: keyType=RSA, storeFileType=JKS] Thread=23 (was 21) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=332 (was 332), ProcessCount=11 (was 11), AvailableMemoryMB=2682 (was 2717) 2025-01-12T17:46:31,474 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[1: keyType=RSA, storeFileType=PEM] Thread=23, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=332, ProcessCount=11, AvailableMemoryMB=2682 2025-01-12T17:46:31,547 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-12T17:46:31,548 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:33107 2025-01-12T17:46:31,548 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-12T17:46:31,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27b01163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:31,550 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:31,551 DEBUG [RPCClient-NioEventLoopGroup-3-5 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:31,578 DEBUG [RPCClient-NioEventLoopGroup-3-5 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:31,579 DEBUG [FileChangeWatcher-hbase_test_key9130784503530308355.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key9130784503530308355.pem thread started 2025-01-12T17:46:31,579 DEBUG [FileChangeWatcher-hbase_test_ca16577311999177518896.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca16577311999177518896.pem thread started 2025-01-12T17:46:31,580 DEBUG [RPCClient-NioEventLoopGroup-3-5 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:31,590 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:31,591 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:31,592 DEBUG [FileChangeWatcher-hbase_test_key9130784503530308355.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key9130784503530308355.pem thread started 2025-01-12T17:46:31,592 DEBUG [FileChangeWatcher-hbase_test_ca16577311999177518896.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca16577311999177518896.pem thread started 2025-01-12T17:46:31,593 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xfadc20e8, L:/127.0.0.1:33107 - R:/127.0.0.1:43820] 2025-01-12T17:46:31,610 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43820, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:32,008 INFO [Time-limited test {}] ipc.AbstractRpcClient(478): The server on localhost,33107,0 is dead - stopping the connection localhost:33107/TestProtobufRpcProto/jenkins (auth:SIMPLE) 2025-01-12T17:46:32,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:32,010 DEBUG [RPCClient-NioEventLoopGroup-3-6 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:32,010 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x20e802f7, L:/127.0.0.1:33107 - R:/127.0.0.1:43836] 2025-01-12T17:46:32,021 INFO [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43836, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:32,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:32,023 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:33107 2025-01-12T17:46:32,024 DEBUG [FileChangeWatcher-hbase_test_key9130784503530308355.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:32,024 DEBUG [FileChangeWatcher-hbase_test_key9130784503530308355.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:32,024 DEBUG [FileChangeWatcher-hbase_test_key9130784503530308355.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key9130784503530308355.pem thread finished 2025-01-12T17:46:32,024 DEBUG [FileChangeWatcher-hbase_test_key9130784503530308355.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key9130784503530308355.pem thread finished 2025-01-12T17:46:32,023 DEBUG [FileChangeWatcher-hbase_test_ca16577311999177518896.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:32,025 DEBUG [FileChangeWatcher-hbase_test_ca16577311999177518896.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca16577311999177518896.pem thread finished 2025-01-12T17:46:32,030 DEBUG [FileChangeWatcher-hbase_test_ca16577311999177518896.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:32,031 DEBUG [FileChangeWatcher-hbase_test_ca16577311999177518896.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca16577311999177518896.pem thread finished 2025-01-12T17:46:32,044 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[1: keyType=RSA, storeFileType=PEM] Thread=26 (was 23) Potentially hanging thread: FileChangeWatcher-hbase_test_ca16577311999177518896.pem java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) app//org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=332 (was 332), ProcessCount=11 (was 11), AvailableMemoryMB=2592 (was 2682) 2025-01-12T17:46:32,052 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[1: keyType=RSA, storeFileType=PEM] Thread=25, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=332, ProcessCount=11, AvailableMemoryMB=2590 2025-01-12T17:46:32,053 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-12T17:46:32,054 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:33407 2025-01-12T17:46:32,054 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-12T17:46:32,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38633be9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:32,055 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:32,057 DEBUG [RPCClient-NioEventLoopGroup-3-7 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:32,062 DEBUG [RPCClient-NioEventLoopGroup-3-7 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:32,062 DEBUG [FileChangeWatcher-hbase_test_key9130784503530308355.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key9130784503530308355.pem thread started 2025-01-12T17:46:32,063 DEBUG [RPCClient-NioEventLoopGroup-3-7 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:32,065 DEBUG [FileChangeWatcher-hbase_test_ca16577311999177518896.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca16577311999177518896.pem thread started 2025-01-12T17:46:32,068 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:32,069 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:32,071 DEBUG [FileChangeWatcher-hbase_test_key9130784503530308355.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key9130784503530308355.pem thread started 2025-01-12T17:46:32,071 DEBUG [FileChangeWatcher-hbase_test_ca16577311999177518896.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca16577311999177518896.pem thread started 2025-01-12T17:46:32,071 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x2fcdf7a9, L:/127.0.0.1:33407 - R:/127.0.0.1:59452] 2025-01-12T17:46:32,086 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59452, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:32,088 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:32,089 DEBUG [FileChangeWatcher-hbase_test_ca16577311999177518896.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:32,089 DEBUG [FileChangeWatcher-hbase_test_key9130784503530308355.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:32,089 DEBUG [FileChangeWatcher-hbase_test_ca16577311999177518896.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca16577311999177518896.pem thread finished 2025-01-12T17:46:32,089 DEBUG [FileChangeWatcher-hbase_test_key9130784503530308355.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key9130784503530308355.pem thread finished 2025-01-12T17:46:32,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba59fb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:32,558 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:32,559 DEBUG [RPCClient-NioEventLoopGroup-3-8 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:32,565 DEBUG [RPCClient-NioEventLoopGroup-3-8 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:32,565 DEBUG [FileChangeWatcher-hbase_test_key9130784503530308355.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key9130784503530308355.pem thread started 2025-01-12T17:46:32,566 DEBUG [FileChangeWatcher-hbase_test_ca16577311999177518896.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca16577311999177518896.pem thread started 2025-01-12T17:46:32,566 DEBUG [RPCClient-NioEventLoopGroup-3-8 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:32,567 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xb340b1fc, L:/127.0.0.1:33407 - R:/127.0.0.1:59462] 2025-01-12T17:46:32,576 WARN [RPCClient-NioEventLoopGroup-3-8 {}] ipc.NettyRpcConnection$2(410): Exception encountered while connecting to the server localhost:33407 javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more 2025-01-12T17:46:32,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:32,578 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:33407 2025-01-12T17:46:32,578 DEBUG [RPCClient-NioEventLoopGroup-3-8 {}] ipc.FailedServers(52): Added failed server with address localhost:33407 to list caused by javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed 2025-01-12T17:46:32,578 DEBUG [FileChangeWatcher-hbase_test_ca16577311999177518896.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:32,578 DEBUG [FileChangeWatcher-hbase_test_key9130784503530308355.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:32,579 DEBUG [FileChangeWatcher-hbase_test_key9130784503530308355.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:32,579 DEBUG [FileChangeWatcher-hbase_test_ca16577311999177518896.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca16577311999177518896.pem thread finished 2025-01-12T17:46:32,579 DEBUG [FileChangeWatcher-hbase_test_key9130784503530308355.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key9130784503530308355.pem thread finished 2025-01-12T17:46:32,579 DEBUG [FileChangeWatcher-hbase_test_key9130784503530308355.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key9130784503530308355.pem thread finished 2025-01-12T17:46:32,578 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(470): Could not get peer certificate on TLS connection from /127.0.0.1:59462, although one is required javax.net.ssl.SSLPeerUnverifiedException: peer not authenticated at sun.security.ssl.SSLSessionImpl.getPeerCertificates(SSLSessionImpl.java:1034) ~[?:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.sslHandshakeCompleteHandler(NettyRpcServer.java:456) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.lambda$initSSL$0(NettyRpcServer.java:446) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.handleUnwrapThrowable(SslHandler.java:1381) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1363) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2025-01-12T17:46:32,579 DEBUG [RPCClient-NioEventLoopGroup-3-8 {}] ipc.BufferCallBeforeInitHandler(140): got ssl exception, which should have already been proceeded, log it here to prevent it being passed to netty's TailContext and trigger a confusing WARN message org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more 2025-01-12T17:46:32,579 WARN [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServerPreambleHandler(100): Connection /127.0.0.1:59462; caught unexpected downstream exception. org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.Alert.createSSLException(Alert.java:117) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:365) ~[?:?] at sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:293) ~[?:?] at sun.security.ssl.TransportContext.dispatch(TransportContext.java:204) ~[?:?] at sun.security.ssl.SSLTransport.decode(SSLTransport.java:172) ~[?:?] at sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:736) ~[?:?] at sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:691) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:506) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:482) ~[?:?] at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:679) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:308) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1465) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 15 more 2025-01-12T17:46:32,580 DEBUG [FileChangeWatcher-hbase_test_ca16577311999177518896.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:32,580 DEBUG [FileChangeWatcher-hbase_test_ca16577311999177518896.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca16577311999177518896.pem thread finished 2025-01-12T17:46:32,598 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[1: keyType=RSA, storeFileType=PEM] Thread=27 (was 25) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=332 (was 332), ProcessCount=11 (was 11), AvailableMemoryMB=2523 (was 2590) 2025-01-12T17:46:32,609 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[2: keyType=RSA, storeFileType=PKCS12] Thread=27, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=332, ProcessCount=11, AvailableMemoryMB=2520 2025-01-12T17:46:32,666 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-12T17:46:32,667 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:34969 2025-01-12T17:46:32,667 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-12T17:46:32,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a79591e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:32,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:32,669 DEBUG [RPCClient-NioEventLoopGroup-3-9 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:32,702 DEBUG [RPCClient-NioEventLoopGroup-3-9 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:32,703 DEBUG [FileChangeWatcher-hbase_test_key10402175833844297132.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key10402175833844297132.p12 thread started 2025-01-12T17:46:32,703 DEBUG [FileChangeWatcher-hbase_test_ca2214503307805643623.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca2214503307805643623.p12 thread started 2025-01-12T17:46:32,703 DEBUG [RPCClient-NioEventLoopGroup-3-9 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:32,721 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:32,737 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:32,739 DEBUG [FileChangeWatcher-hbase_test_key10402175833844297132.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key10402175833844297132.p12 thread started 2025-01-12T17:46:32,739 DEBUG [FileChangeWatcher-hbase_test_ca2214503307805643623.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca2214503307805643623.p12 thread started 2025-01-12T17:46:32,739 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xfd42497b, L:/127.0.0.1:34969 - R:/127.0.0.1:42196] 2025-01-12T17:46:32,793 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42196, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:33,266 INFO [Time-limited test {}] ipc.AbstractRpcClient(478): The server on localhost,34969,0 is dead - stopping the connection localhost:34969/TestProtobufRpcProto/jenkins (auth:SIMPLE) 2025-01-12T17:46:33,267 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:33,268 DEBUG [RPCClient-NioEventLoopGroup-3-10 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:33,269 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xa15eb9d5, L:/127.0.0.1:34969 - R:/127.0.0.1:42202] 2025-01-12T17:46:33,277 INFO [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42202, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:33,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:33,279 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:34969 2025-01-12T17:46:33,279 DEBUG [FileChangeWatcher-hbase_test_ca2214503307805643623.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:33,279 DEBUG [FileChangeWatcher-hbase_test_key10402175833844297132.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:33,279 DEBUG [FileChangeWatcher-hbase_test_ca2214503307805643623.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:33,280 DEBUG [FileChangeWatcher-hbase_test_ca2214503307805643623.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca2214503307805643623.p12 thread finished 2025-01-12T17:46:33,280 DEBUG [FileChangeWatcher-hbase_test_key10402175833844297132.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key10402175833844297132.p12 thread finished 2025-01-12T17:46:33,280 DEBUG [FileChangeWatcher-hbase_test_ca2214503307805643623.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca2214503307805643623.p12 thread finished 2025-01-12T17:46:33,279 DEBUG [FileChangeWatcher-hbase_test_key10402175833844297132.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:33,280 DEBUG [FileChangeWatcher-hbase_test_key10402175833844297132.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key10402175833844297132.p12 thread finished 2025-01-12T17:46:33,287 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[2: keyType=RSA, storeFileType=PKCS12] Thread=29 (was 27) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=321 (was 332), ProcessCount=11 (was 11), AvailableMemoryMB=2442 (was 2520) 2025-01-12T17:46:33,293 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[2: keyType=RSA, storeFileType=PKCS12] Thread=29, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=321, ProcessCount=11, AvailableMemoryMB=2435 2025-01-12T17:46:33,294 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-12T17:46:33,295 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:43729 2025-01-12T17:46:33,295 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-12T17:46:33,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c725c2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:33,296 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:33,297 DEBUG [RPCClient-NioEventLoopGroup-3-11 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:33,328 DEBUG [RPCClient-NioEventLoopGroup-3-11 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:33,329 DEBUG [FileChangeWatcher-hbase_test_key10402175833844297132.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key10402175833844297132.p12 thread started 2025-01-12T17:46:33,329 DEBUG [FileChangeWatcher-hbase_test_ca2214503307805643623.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca2214503307805643623.p12 thread started 2025-01-12T17:46:33,329 DEBUG [RPCClient-NioEventLoopGroup-3-11 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:33,345 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:33,363 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:33,363 DEBUG [FileChangeWatcher-hbase_test_key10402175833844297132.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key10402175833844297132.p12 thread started 2025-01-12T17:46:33,364 DEBUG [FileChangeWatcher-hbase_test_ca2214503307805643623.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca2214503307805643623.p12 thread started 2025-01-12T17:46:33,364 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x31cacdc5, L:/127.0.0.1:43729 - R:/127.0.0.1:39860] 2025-01-12T17:46:33,406 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39860, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:33,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:33,408 DEBUG [FileChangeWatcher-hbase_test_key10402175833844297132.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:33,408 DEBUG [FileChangeWatcher-hbase_test_ca2214503307805643623.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:33,409 DEBUG [FileChangeWatcher-hbase_test_key10402175833844297132.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key10402175833844297132.p12 thread finished 2025-01-12T17:46:33,409 DEBUG [FileChangeWatcher-hbase_test_ca2214503307805643623.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca2214503307805643623.p12 thread finished 2025-01-12T17:46:33,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78efc1e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:33,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:33,761 DEBUG [RPCClient-NioEventLoopGroup-3-12 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:33,792 DEBUG [RPCClient-NioEventLoopGroup-3-12 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:33,793 DEBUG [FileChangeWatcher-hbase_test_key10402175833844297132.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key10402175833844297132.p12 thread started 2025-01-12T17:46:33,793 DEBUG [FileChangeWatcher-hbase_test_ca2214503307805643623.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca2214503307805643623.p12 thread started 2025-01-12T17:46:33,793 DEBUG [RPCClient-NioEventLoopGroup-3-12 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:33,794 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xaf30764e, L:/127.0.0.1:43729 - R:/127.0.0.1:39872] 2025-01-12T17:46:33,812 WARN [RPCClient-NioEventLoopGroup-3-12 {}] ipc.NettyRpcConnection$2(410): Exception encountered while connecting to the server localhost:43729 javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more 2025-01-12T17:46:33,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:33,814 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:43729 2025-01-12T17:46:33,814 DEBUG [RPCClient-NioEventLoopGroup-3-12 {}] ipc.FailedServers(52): Added failed server with address localhost:43729 to list caused by javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed 2025-01-12T17:46:33,814 DEBUG [FileChangeWatcher-hbase_test_ca2214503307805643623.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:33,814 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(470): Could not get peer certificate on TLS connection from /127.0.0.1:39872, although one is required javax.net.ssl.SSLPeerUnverifiedException: peer not authenticated at sun.security.ssl.SSLSessionImpl.getPeerCertificates(SSLSessionImpl.java:1034) ~[?:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.sslHandshakeCompleteHandler(NettyRpcServer.java:456) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.lambda$initSSL$0(NettyRpcServer.java:446) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.handleUnwrapThrowable(SslHandler.java:1381) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1363) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2025-01-12T17:46:33,814 DEBUG [FileChangeWatcher-hbase_test_key10402175833844297132.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:33,814 DEBUG [FileChangeWatcher-hbase_test_ca2214503307805643623.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca2214503307805643623.p12 thread finished 2025-01-12T17:46:33,814 DEBUG [FileChangeWatcher-hbase_test_ca2214503307805643623.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:33,815 DEBUG [FileChangeWatcher-hbase_test_key10402175833844297132.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key10402175833844297132.p12 thread finished 2025-01-12T17:46:33,814 DEBUG [FileChangeWatcher-hbase_test_key10402175833844297132.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:33,815 DEBUG [FileChangeWatcher-hbase_test_ca2214503307805643623.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca2214503307805643623.p12 thread finished 2025-01-12T17:46:33,815 DEBUG [FileChangeWatcher-hbase_test_key10402175833844297132.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key10402175833844297132.p12 thread finished 2025-01-12T17:46:33,815 WARN [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServerPreambleHandler(100): Connection /127.0.0.1:39872; caught unexpected downstream exception. org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.Alert.createSSLException(Alert.java:117) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:365) ~[?:?] at sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:293) ~[?:?] at sun.security.ssl.TransportContext.dispatch(TransportContext.java:204) ~[?:?] at sun.security.ssl.SSLTransport.decode(SSLTransport.java:172) ~[?:?] at sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:736) ~[?:?] at sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:691) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:506) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:482) ~[?:?] at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:679) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:308) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1465) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 15 more 2025-01-12T17:46:33,815 DEBUG [RPCClient-NioEventLoopGroup-3-12 {}] ipc.BufferCallBeforeInitHandler(140): got ssl exception, which should have already been proceeded, log it here to prevent it being passed to netty's TailContext and trigger a confusing WARN message org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more 2025-01-12T17:46:33,824 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[2: keyType=RSA, storeFileType=PKCS12] Thread=31 (was 29) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=321 (was 321), ProcessCount=11 (was 11), AvailableMemoryMB=2369 (was 2435) 2025-01-12T17:46:33,831 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[3: keyType=RSA, storeFileType=BCFKS] Thread=31, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=321, ProcessCount=11, AvailableMemoryMB=2377 2025-01-12T17:46:34,366 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-12T17:46:34,367 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:37001 2025-01-12T17:46:34,367 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-12T17:46:34,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b1c80e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:34,368 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:34,369 DEBUG [RPCClient-NioEventLoopGroup-3-13 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:34,704 DEBUG [RPCClient-NioEventLoopGroup-3-13 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:34,705 DEBUG [FileChangeWatcher-hbase_test_key18296761770950611839.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key18296761770950611839.bcfks thread started 2025-01-12T17:46:34,705 DEBUG [FileChangeWatcher-hbase_test_ca233368039109218949.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca233368039109218949.bcfks thread started 2025-01-12T17:46:34,705 DEBUG [RPCClient-NioEventLoopGroup-3-13 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:34,849 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:34,993 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:34,994 DEBUG [FileChangeWatcher-hbase_test_key18296761770950611839.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key18296761770950611839.bcfks thread started 2025-01-12T17:46:34,994 DEBUG [FileChangeWatcher-hbase_test_ca233368039109218949.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca233368039109218949.bcfks thread started 2025-01-12T17:46:34,994 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x024f8f17, L:/127.0.0.1:37001 - R:/127.0.0.1:48774] 2025-01-12T17:46:35,189 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48774, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:35,779 INFO [Time-limited test {}] ipc.AbstractRpcClient(478): The server on localhost,37001,0 is dead - stopping the connection localhost:37001/TestProtobufRpcProto/jenkins (auth:SIMPLE) 2025-01-12T17:46:35,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:35,781 DEBUG [RPCClient-NioEventLoopGroup-3-14 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:35,784 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xb8f8d8ac, L:/127.0.0.1:37001 - R:/127.0.0.1:48790] 2025-01-12T17:46:35,790 INFO [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48790, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:35,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:35,792 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:37001 2025-01-12T17:46:35,792 DEBUG [FileChangeWatcher-hbase_test_key18296761770950611839.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:35,792 DEBUG [FileChangeWatcher-hbase_test_ca233368039109218949.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:35,792 DEBUG [FileChangeWatcher-hbase_test_key18296761770950611839.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:35,792 DEBUG [FileChangeWatcher-hbase_test_ca233368039109218949.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca233368039109218949.bcfks thread finished 2025-01-12T17:46:35,792 DEBUG [FileChangeWatcher-hbase_test_key18296761770950611839.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key18296761770950611839.bcfks thread finished 2025-01-12T17:46:35,792 DEBUG [FileChangeWatcher-hbase_test_ca233368039109218949.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:35,793 DEBUG [FileChangeWatcher-hbase_test_ca233368039109218949.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca233368039109218949.bcfks thread finished 2025-01-12T17:46:35,793 DEBUG [FileChangeWatcher-hbase_test_key18296761770950611839.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key18296761770950611839.bcfks thread finished 2025-01-12T17:46:35,800 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[3: keyType=RSA, storeFileType=BCFKS] Thread=33 (was 31) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=321 (was 321), ProcessCount=11 (was 11), AvailableMemoryMB=2371 (was 2377) 2025-01-12T17:46:35,807 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[3: keyType=RSA, storeFileType=BCFKS] Thread=33, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=321, ProcessCount=11, AvailableMemoryMB=2371 2025-01-12T17:46:35,808 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-12T17:46:35,809 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:40691 2025-01-12T17:46:35,809 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-12T17:46:35,810 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@522006d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:35,810 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:35,811 DEBUG [RPCClient-NioEventLoopGroup-3-15 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:36,114 DEBUG [RPCClient-NioEventLoopGroup-3-15 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:36,115 DEBUG [FileChangeWatcher-hbase_test_key18296761770950611839.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key18296761770950611839.bcfks thread started 2025-01-12T17:46:36,115 DEBUG [FileChangeWatcher-hbase_test_ca233368039109218949.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca233368039109218949.bcfks thread started 2025-01-12T17:46:36,115 DEBUG [RPCClient-NioEventLoopGroup-3-15 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:36,266 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:36,415 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:36,416 DEBUG [FileChangeWatcher-hbase_test_key18296761770950611839.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key18296761770950611839.bcfks thread started 2025-01-12T17:46:36,416 DEBUG [FileChangeWatcher-hbase_test_ca233368039109218949.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca233368039109218949.bcfks thread started 2025-01-12T17:46:36,416 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x99663d44, L:/127.0.0.1:40691 - R:/127.0.0.1:59834] 2025-01-12T17:46:36,635 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59834, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:36,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:36,638 DEBUG [FileChangeWatcher-hbase_test_key18296761770950611839.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:36,638 DEBUG [FileChangeWatcher-hbase_test_ca233368039109218949.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:36,638 DEBUG [FileChangeWatcher-hbase_test_ca233368039109218949.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca233368039109218949.bcfks thread finished 2025-01-12T17:46:36,638 DEBUG [FileChangeWatcher-hbase_test_key18296761770950611839.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key18296761770950611839.bcfks thread finished 2025-01-12T17:46:37,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23414969, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:37,521 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:37,522 DEBUG [RPCClient-NioEventLoopGroup-3-16 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:37,818 DEBUG [RPCClient-NioEventLoopGroup-3-16 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:37,819 DEBUG [FileChangeWatcher-hbase_test_key18296761770950611839.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key18296761770950611839.bcfks thread started 2025-01-12T17:46:37,819 DEBUG [FileChangeWatcher-hbase_test_ca233368039109218949.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca233368039109218949.bcfks thread started 2025-01-12T17:46:37,819 DEBUG [RPCClient-NioEventLoopGroup-3-16 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:37,820 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x20ba5b75, L:/127.0.0.1:40691 - R:/127.0.0.1:59846] 2025-01-12T17:46:37,826 WARN [RPCClient-NioEventLoopGroup-3-16 {}] ipc.NettyRpcConnection$2(410): Exception encountered while connecting to the server localhost:40691 javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more 2025-01-12T17:46:37,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:37,827 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:40691 2025-01-12T17:46:37,827 DEBUG [FileChangeWatcher-hbase_test_key18296761770950611839.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:37,828 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(470): Could not get peer certificate on TLS connection from /127.0.0.1:59846, although one is required javax.net.ssl.SSLPeerUnverifiedException: peer not authenticated at sun.security.ssl.SSLSessionImpl.getPeerCertificates(SSLSessionImpl.java:1034) ~[?:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.sslHandshakeCompleteHandler(NettyRpcServer.java:456) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.lambda$initSSL$0(NettyRpcServer.java:446) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.handleUnwrapThrowable(SslHandler.java:1381) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1363) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2025-01-12T17:46:37,827 DEBUG [FileChangeWatcher-hbase_test_ca233368039109218949.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:37,828 DEBUG [FileChangeWatcher-hbase_test_key18296761770950611839.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:37,828 DEBUG [FileChangeWatcher-hbase_test_key18296761770950611839.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key18296761770950611839.bcfks thread finished 2025-01-12T17:46:37,828 DEBUG [FileChangeWatcher-hbase_test_ca233368039109218949.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:37,828 DEBUG [RPCClient-NioEventLoopGroup-3-16 {}] ipc.FailedServers(52): Added failed server with address localhost:40691 to list caused by javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed 2025-01-12T17:46:37,828 DEBUG [FileChangeWatcher-hbase_test_ca233368039109218949.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca233368039109218949.bcfks thread finished 2025-01-12T17:46:37,828 DEBUG [FileChangeWatcher-hbase_test_key18296761770950611839.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key18296761770950611839.bcfks thread finished 2025-01-12T17:46:37,828 DEBUG [FileChangeWatcher-hbase_test_ca233368039109218949.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca233368039109218949.bcfks thread finished 2025-01-12T17:46:37,829 WARN [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServerPreambleHandler(100): Connection /127.0.0.1:59846; caught unexpected downstream exception. org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.Alert.createSSLException(Alert.java:117) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:365) ~[?:?] at sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:293) ~[?:?] at sun.security.ssl.TransportContext.dispatch(TransportContext.java:204) ~[?:?] at sun.security.ssl.SSLTransport.decode(SSLTransport.java:172) ~[?:?] at sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:736) ~[?:?] at sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:691) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:506) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:482) ~[?:?] at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:679) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:308) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1465) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 15 more 2025-01-12T17:46:37,829 DEBUG [RPCClient-NioEventLoopGroup-3-16 {}] ipc.BufferCallBeforeInitHandler(140): got ssl exception, which should have already been proceeded, log it here to prevent it being passed to netty's TailContext and trigger a confusing WARN message org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more 2025-01-12T17:46:37,837 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[3: keyType=RSA, storeFileType=BCFKS] Thread=35 (was 33) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=312 (was 321), ProcessCount=11 (was 11), AvailableMemoryMB=2363 (was 2371) 2025-01-12T17:46:37,843 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[4: keyType=EC, storeFileType=JKS] Thread=35, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=312, ProcessCount=11, AvailableMemoryMB=2361 2025-01-12T17:46:37,987 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-12T17:46:37,988 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:32945 2025-01-12T17:46:37,988 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-12T17:46:37,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a2ba41c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:37,989 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:37,990 DEBUG [RPCClient-NioEventLoopGroup-3-1 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:38,020 DEBUG [RPCClient-NioEventLoopGroup-3-1 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:38,021 DEBUG [FileChangeWatcher-hbase_test_key17788752034570797194.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key17788752034570797194.jks thread started 2025-01-12T17:46:38,021 DEBUG [FileChangeWatcher-hbase_test_ca1281393680828501297.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca1281393680828501297.jks thread started 2025-01-12T17:46:38,021 DEBUG [RPCClient-NioEventLoopGroup-3-1 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:38,038 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:38,054 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:38,054 DEBUG [FileChangeWatcher-hbase_test_key17788752034570797194.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key17788752034570797194.jks thread started 2025-01-12T17:46:38,055 DEBUG [FileChangeWatcher-hbase_test_ca1281393680828501297.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca1281393680828501297.jks thread started 2025-01-12T17:46:38,055 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xffb2115e, L:/127.0.0.1:32945 - R:/127.0.0.1:35372] 2025-01-12T17:46:38,113 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35372, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:38,165 INFO [Time-limited test {}] ipc.AbstractRpcClient(478): The server on localhost,32945,0 is dead - stopping the connection localhost:32945/TestProtobufRpcProto/jenkins (auth:SIMPLE) 2025-01-12T17:46:38,165 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:38,166 DEBUG [RPCClient-NioEventLoopGroup-3-2 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:38,167 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xe35ff6f2, L:/127.0.0.1:32945 - R:/127.0.0.1:35386] 2025-01-12T17:46:38,172 INFO [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35386, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:38,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:38,174 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:32945 2025-01-12T17:46:38,174 DEBUG [FileChangeWatcher-hbase_test_key17788752034570797194.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,174 DEBUG [FileChangeWatcher-hbase_test_key17788752034570797194.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key17788752034570797194.jks thread finished 2025-01-12T17:46:38,174 DEBUG [FileChangeWatcher-hbase_test_key17788752034570797194.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,174 DEBUG [FileChangeWatcher-hbase_test_ca1281393680828501297.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,174 DEBUG [FileChangeWatcher-hbase_test_key17788752034570797194.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key17788752034570797194.jks thread finished 2025-01-12T17:46:38,174 DEBUG [FileChangeWatcher-hbase_test_ca1281393680828501297.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca1281393680828501297.jks thread finished 2025-01-12T17:46:38,174 DEBUG [FileChangeWatcher-hbase_test_ca1281393680828501297.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,174 DEBUG [FileChangeWatcher-hbase_test_ca1281393680828501297.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca1281393680828501297.jks thread finished 2025-01-12T17:46:38,184 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[4: keyType=EC, storeFileType=JKS] Thread=35 (was 35), OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=312 (was 312), ProcessCount=11 (was 11), AvailableMemoryMB=2379 (was 2361) - AvailableMemoryMB LEAK? - 2025-01-12T17:46:38,194 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[4: keyType=EC, storeFileType=JKS] Thread=35, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=312, ProcessCount=11, AvailableMemoryMB=2378 2025-01-12T17:46:38,195 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-12T17:46:38,195 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:40301 2025-01-12T17:46:38,195 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-12T17:46:38,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b937049, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:38,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:38,197 DEBUG [RPCClient-NioEventLoopGroup-3-3 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:38,227 DEBUG [RPCClient-NioEventLoopGroup-3-3 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:38,228 DEBUG [FileChangeWatcher-hbase_test_key17788752034570797194.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key17788752034570797194.jks thread started 2025-01-12T17:46:38,228 DEBUG [FileChangeWatcher-hbase_test_ca1281393680828501297.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca1281393680828501297.jks thread started 2025-01-12T17:46:38,228 DEBUG [RPCClient-NioEventLoopGroup-3-3 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:38,244 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:38,259 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:38,259 DEBUG [FileChangeWatcher-hbase_test_key17788752034570797194.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key17788752034570797194.jks thread started 2025-01-12T17:46:38,260 DEBUG [FileChangeWatcher-hbase_test_ca1281393680828501297.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca1281393680828501297.jks thread started 2025-01-12T17:46:38,260 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xfacb5cf1, L:/127.0.0.1:40301 - R:/127.0.0.1:55072] 2025-01-12T17:46:38,306 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55072, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:38,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:38,308 DEBUG [FileChangeWatcher-hbase_test_ca1281393680828501297.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,308 DEBUG [FileChangeWatcher-hbase_test_key17788752034570797194.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,309 DEBUG [FileChangeWatcher-hbase_test_ca1281393680828501297.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca1281393680828501297.jks thread finished 2025-01-12T17:46:38,309 DEBUG [FileChangeWatcher-hbase_test_key17788752034570797194.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key17788752034570797194.jks thread finished 2025-01-12T17:46:38,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@169013e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:38,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:38,384 DEBUG [RPCClient-NioEventLoopGroup-3-4 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:38,414 DEBUG [RPCClient-NioEventLoopGroup-3-4 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:38,415 DEBUG [FileChangeWatcher-hbase_test_key17788752034570797194.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key17788752034570797194.jks thread started 2025-01-12T17:46:38,415 DEBUG [RPCClient-NioEventLoopGroup-3-4 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:38,415 DEBUG [FileChangeWatcher-hbase_test_ca1281393680828501297.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca1281393680828501297.jks thread started 2025-01-12T17:46:38,416 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xc93f3f5f, L:/127.0.0.1:40301 - R:/127.0.0.1:55076] 2025-01-12T17:46:38,433 WARN [RPCClient-NioEventLoopGroup-3-4 {}] ipc.NettyRpcConnection$2(410): Exception encountered while connecting to the server localhost:40301 javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more 2025-01-12T17:46:38,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:38,434 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:40301 2025-01-12T17:46:38,434 DEBUG [RPCClient-NioEventLoopGroup-3-4 {}] ipc.FailedServers(52): Added failed server with address localhost:40301 to list caused by javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed 2025-01-12T17:46:38,434 DEBUG [FileChangeWatcher-hbase_test_ca1281393680828501297.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,434 DEBUG [FileChangeWatcher-hbase_test_key17788752034570797194.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,434 DEBUG [FileChangeWatcher-hbase_test_ca1281393680828501297.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca1281393680828501297.jks thread finished 2025-01-12T17:46:38,434 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(470): Could not get peer certificate on TLS connection from /127.0.0.1:55076, although one is required javax.net.ssl.SSLPeerUnverifiedException: peer not authenticated at sun.security.ssl.SSLSessionImpl.getPeerCertificates(SSLSessionImpl.java:1034) ~[?:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.sslHandshakeCompleteHandler(NettyRpcServer.java:456) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.lambda$initSSL$0(NettyRpcServer.java:446) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.handleUnwrapThrowable(SslHandler.java:1381) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1363) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2025-01-12T17:46:38,434 DEBUG [FileChangeWatcher-hbase_test_ca1281393680828501297.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,434 DEBUG [FileChangeWatcher-hbase_test_key17788752034570797194.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key17788752034570797194.jks thread finished 2025-01-12T17:46:38,434 DEBUG [FileChangeWatcher-hbase_test_key17788752034570797194.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,434 DEBUG [FileChangeWatcher-hbase_test_ca1281393680828501297.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca1281393680828501297.jks thread finished 2025-01-12T17:46:38,434 DEBUG [FileChangeWatcher-hbase_test_key17788752034570797194.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key17788752034570797194.jks thread finished 2025-01-12T17:46:38,434 DEBUG [RPCClient-NioEventLoopGroup-3-4 {}] ipc.BufferCallBeforeInitHandler(140): got ssl exception, which should have already been proceeded, log it here to prevent it being passed to netty's TailContext and trigger a confusing WARN message org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more 2025-01-12T17:46:38,435 WARN [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServerPreambleHandler(100): Connection /127.0.0.1:55076; caught unexpected downstream exception. org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.Alert.createSSLException(Alert.java:117) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:365) ~[?:?] at sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:293) ~[?:?] at sun.security.ssl.TransportContext.dispatch(TransportContext.java:204) ~[?:?] at sun.security.ssl.SSLTransport.decode(SSLTransport.java:172) ~[?:?] at sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:736) ~[?:?] at sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:691) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:506) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:482) ~[?:?] at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:679) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:308) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1465) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 15 more 2025-01-12T17:46:38,444 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[4: keyType=EC, storeFileType=JKS] Thread=35 (was 35), OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=312 (was 312), ProcessCount=11 (was 11), AvailableMemoryMB=2372 (was 2378) 2025-01-12T17:46:38,451 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[5: keyType=EC, storeFileType=PEM] Thread=35, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=312, ProcessCount=11, AvailableMemoryMB=2371 2025-01-12T17:46:38,457 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-12T17:46:38,458 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:43493 2025-01-12T17:46:38,458 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-12T17:46:38,458 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30de7cd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:38,459 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:38,460 DEBUG [RPCClient-NioEventLoopGroup-3-5 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:38,463 DEBUG [RPCClient-NioEventLoopGroup-3-5 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:38,464 DEBUG [FileChangeWatcher-hbase_test_key14812549897044507855.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key14812549897044507855.pem thread started 2025-01-12T17:46:38,464 DEBUG [FileChangeWatcher-hbase_test_ca7811837285697992755.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca7811837285697992755.pem thread started 2025-01-12T17:46:38,464 DEBUG [RPCClient-NioEventLoopGroup-3-5 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:38,468 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:38,468 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:38,469 DEBUG [FileChangeWatcher-hbase_test_key14812549897044507855.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key14812549897044507855.pem thread started 2025-01-12T17:46:38,469 DEBUG [FileChangeWatcher-hbase_test_ca7811837285697992755.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca7811837285697992755.pem thread started 2025-01-12T17:46:38,469 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x0442b140, L:/127.0.0.1:43493 - R:/127.0.0.1:40614] 2025-01-12T17:46:38,491 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40614, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:38,502 INFO [Time-limited test {}] ipc.AbstractRpcClient(478): The server on localhost,43493,0 is dead - stopping the connection localhost:43493/TestProtobufRpcProto/jenkins (auth:SIMPLE) 2025-01-12T17:46:38,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:38,503 DEBUG [RPCClient-NioEventLoopGroup-3-6 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:38,504 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x4656dde5, L:/127.0.0.1:43493 - R:/127.0.0.1:40616] 2025-01-12T17:46:38,508 INFO [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40616, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:38,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:38,510 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:43493 2025-01-12T17:46:38,510 DEBUG [FileChangeWatcher-hbase_test_ca7811837285697992755.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,510 DEBUG [FileChangeWatcher-hbase_test_key14812549897044507855.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,510 DEBUG [FileChangeWatcher-hbase_test_key14812549897044507855.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,510 DEBUG [FileChangeWatcher-hbase_test_ca7811837285697992755.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca7811837285697992755.pem thread finished 2025-01-12T17:46:38,510 DEBUG [FileChangeWatcher-hbase_test_key14812549897044507855.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key14812549897044507855.pem thread finished 2025-01-12T17:46:38,510 DEBUG [FileChangeWatcher-hbase_test_key14812549897044507855.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key14812549897044507855.pem thread finished 2025-01-12T17:46:38,510 DEBUG [FileChangeWatcher-hbase_test_ca7811837285697992755.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,510 DEBUG [FileChangeWatcher-hbase_test_ca7811837285697992755.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca7811837285697992755.pem thread finished 2025-01-12T17:46:38,518 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[5: keyType=EC, storeFileType=PEM] Thread=35 (was 35), OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=312 (was 312), ProcessCount=11 (was 11), AvailableMemoryMB=2369 (was 2371) 2025-01-12T17:46:38,525 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[5: keyType=EC, storeFileType=PEM] Thread=35, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=312, ProcessCount=11, AvailableMemoryMB=2369 2025-01-12T17:46:38,526 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-12T17:46:38,527 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:36791 2025-01-12T17:46:38,527 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-12T17:46:38,527 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@274a80f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:38,527 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:38,528 DEBUG [RPCClient-NioEventLoopGroup-3-7 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:38,532 DEBUG [RPCClient-NioEventLoopGroup-3-7 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:38,533 DEBUG [FileChangeWatcher-hbase_test_key14812549897044507855.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key14812549897044507855.pem thread started 2025-01-12T17:46:38,533 DEBUG [FileChangeWatcher-hbase_test_ca7811837285697992755.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca7811837285697992755.pem thread started 2025-01-12T17:46:38,533 DEBUG [RPCClient-NioEventLoopGroup-3-7 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:38,537 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:38,537 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:38,538 DEBUG [FileChangeWatcher-hbase_test_key14812549897044507855.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key14812549897044507855.pem thread started 2025-01-12T17:46:38,538 DEBUG [FileChangeWatcher-hbase_test_ca7811837285697992755.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca7811837285697992755.pem thread started 2025-01-12T17:46:38,538 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xa34afc9f, L:/127.0.0.1:36791 - R:/127.0.0.1:58128] 2025-01-12T17:46:38,555 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58128, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:38,557 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:38,557 DEBUG [FileChangeWatcher-hbase_test_key14812549897044507855.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,557 DEBUG [FileChangeWatcher-hbase_test_ca7811837285697992755.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,557 DEBUG [FileChangeWatcher-hbase_test_key14812549897044507855.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key14812549897044507855.pem thread finished 2025-01-12T17:46:38,558 DEBUG [FileChangeWatcher-hbase_test_ca7811837285697992755.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca7811837285697992755.pem thread finished 2025-01-12T17:46:38,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a37f0e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:38,566 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:38,567 DEBUG [RPCClient-NioEventLoopGroup-3-8 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:38,570 DEBUG [RPCClient-NioEventLoopGroup-3-8 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:38,571 DEBUG [FileChangeWatcher-hbase_test_key14812549897044507855.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key14812549897044507855.pem thread started 2025-01-12T17:46:38,571 DEBUG [FileChangeWatcher-hbase_test_ca7811837285697992755.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca7811837285697992755.pem thread started 2025-01-12T17:46:38,571 DEBUG [RPCClient-NioEventLoopGroup-3-8 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:38,572 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x8c4023ec, L:/127.0.0.1:36791 - R:/127.0.0.1:58134] 2025-01-12T17:46:38,579 WARN [RPCClient-NioEventLoopGroup-3-8 {}] ipc.NettyRpcConnection$2(410): Exception encountered while connecting to the server localhost:36791 javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more 2025-01-12T17:46:38,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:38,580 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:36791 2025-01-12T17:46:38,580 DEBUG [RPCClient-NioEventLoopGroup-3-8 {}] ipc.FailedServers(52): Added failed server with address localhost:36791 to list caused by javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed 2025-01-12T17:46:38,580 DEBUG [FileChangeWatcher-hbase_test_ca7811837285697992755.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,580 DEBUG [FileChangeWatcher-hbase_test_key14812549897044507855.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,580 DEBUG [FileChangeWatcher-hbase_test_key14812549897044507855.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,580 DEBUG [FileChangeWatcher-hbase_test_key14812549897044507855.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key14812549897044507855.pem thread finished 2025-01-12T17:46:38,580 DEBUG [FileChangeWatcher-hbase_test_ca7811837285697992755.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca7811837285697992755.pem thread finished 2025-01-12T17:46:38,580 DEBUG [FileChangeWatcher-hbase_test_ca7811837285697992755.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,580 DEBUG [FileChangeWatcher-hbase_test_key14812549897044507855.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key14812549897044507855.pem thread finished 2025-01-12T17:46:38,580 DEBUG [FileChangeWatcher-hbase_test_ca7811837285697992755.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca7811837285697992755.pem thread finished 2025-01-12T17:46:38,580 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(470): Could not get peer certificate on TLS connection from /127.0.0.1:58134, although one is required javax.net.ssl.SSLPeerUnverifiedException: peer not authenticated at sun.security.ssl.SSLSessionImpl.getPeerCertificates(SSLSessionImpl.java:1034) ~[?:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.sslHandshakeCompleteHandler(NettyRpcServer.java:456) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.lambda$initSSL$0(NettyRpcServer.java:446) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.handleUnwrapThrowable(SslHandler.java:1381) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1363) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2025-01-12T17:46:38,580 DEBUG [RPCClient-NioEventLoopGroup-3-8 {}] ipc.BufferCallBeforeInitHandler(140): got ssl exception, which should have already been proceeded, log it here to prevent it being passed to netty's TailContext and trigger a confusing WARN message org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more 2025-01-12T17:46:38,582 WARN [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServerPreambleHandler(100): Connection /127.0.0.1:58134; caught unexpected downstream exception. org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.Alert.createSSLException(Alert.java:117) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:365) ~[?:?] at sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:293) ~[?:?] at sun.security.ssl.TransportContext.dispatch(TransportContext.java:204) ~[?:?] at sun.security.ssl.SSLTransport.decode(SSLTransport.java:172) ~[?:?] at sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:736) ~[?:?] at sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:691) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:506) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:482) ~[?:?] at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:679) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:308) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1465) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 15 more 2025-01-12T17:46:38,590 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[5: keyType=EC, storeFileType=PEM] Thread=35 (was 35), OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=312 (was 312), ProcessCount=11 (was 11), AvailableMemoryMB=2367 (was 2369) 2025-01-12T17:46:38,597 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[6: keyType=EC, storeFileType=PKCS12] Thread=35, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=312, ProcessCount=11, AvailableMemoryMB=2367 2025-01-12T17:46:38,641 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-12T17:46:38,642 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:43355 2025-01-12T17:46:38,642 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-12T17:46:38,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bfc3b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:38,643 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:38,644 DEBUG [RPCClient-NioEventLoopGroup-3-9 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:38,673 DEBUG [RPCClient-NioEventLoopGroup-3-9 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:38,674 DEBUG [FileChangeWatcher-hbase_test_key13496763275007779709.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key13496763275007779709.p12 thread started 2025-01-12T17:46:38,674 DEBUG [FileChangeWatcher-hbase_test_ca2349445730224336356.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca2349445730224336356.p12 thread started 2025-01-12T17:46:38,674 DEBUG [RPCClient-NioEventLoopGroup-3-9 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:38,689 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:38,705 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:38,706 DEBUG [FileChangeWatcher-hbase_test_key13496763275007779709.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key13496763275007779709.p12 thread started 2025-01-12T17:46:38,706 DEBUG [FileChangeWatcher-hbase_test_ca2349445730224336356.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca2349445730224336356.p12 thread started 2025-01-12T17:46:38,706 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x2a1ef8bb, L:/127.0.0.1:43355 - R:/127.0.0.1:47772] 2025-01-12T17:46:38,745 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47772, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:38,817 INFO [Time-limited test {}] ipc.AbstractRpcClient(478): The server on localhost,43355,0 is dead - stopping the connection localhost:43355/TestProtobufRpcProto/jenkins (auth:SIMPLE) 2025-01-12T17:46:38,817 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:38,818 DEBUG [RPCClient-NioEventLoopGroup-3-10 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:38,820 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x6abdb28b, L:/127.0.0.1:43355 - R:/127.0.0.1:47780] 2025-01-12T17:46:38,824 INFO [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47780, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:38,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:38,826 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:43355 2025-01-12T17:46:38,826 DEBUG [FileChangeWatcher-hbase_test_key13496763275007779709.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,826 DEBUG [FileChangeWatcher-hbase_test_key13496763275007779709.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key13496763275007779709.p12 thread finished 2025-01-12T17:46:38,826 DEBUG [FileChangeWatcher-hbase_test_key13496763275007779709.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,826 DEBUG [FileChangeWatcher-hbase_test_ca2349445730224336356.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,827 DEBUG [FileChangeWatcher-hbase_test_key13496763275007779709.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key13496763275007779709.p12 thread finished 2025-01-12T17:46:38,827 DEBUG [FileChangeWatcher-hbase_test_ca2349445730224336356.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca2349445730224336356.p12 thread finished 2025-01-12T17:46:38,827 DEBUG [FileChangeWatcher-hbase_test_ca2349445730224336356.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,827 DEBUG [FileChangeWatcher-hbase_test_ca2349445730224336356.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca2349445730224336356.p12 thread finished 2025-01-12T17:46:38,835 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[6: keyType=EC, storeFileType=PKCS12] Thread=36 (was 35) Potentially hanging thread: FifoRpcScheduler.handler-pool-0 java.base@17.0.11/java.lang.Thread.currentThread(Native Method) java.base@17.0.11/java.util.concurrent.locks.ReentrantLock$NonfairSync.tryAcquire(ReentrantLock.java:243) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:672) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1636) java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=312 (was 312), ProcessCount=11 (was 11), AvailableMemoryMB=3876 (was 2367) - AvailableMemoryMB LEAK? - 2025-01-12T17:46:38,842 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[6: keyType=EC, storeFileType=PKCS12] Thread=35, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=312, ProcessCount=11, AvailableMemoryMB=3875 2025-01-12T17:46:38,843 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-12T17:46:38,844 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:45497 2025-01-12T17:46:38,844 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-12T17:46:38,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3279038c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:38,844 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:38,845 DEBUG [RPCClient-NioEventLoopGroup-3-11 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:38,896 DEBUG [RPCClient-NioEventLoopGroup-3-11 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:38,897 DEBUG [FileChangeWatcher-hbase_test_key13496763275007779709.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key13496763275007779709.p12 thread started 2025-01-12T17:46:38,897 DEBUG [FileChangeWatcher-hbase_test_ca2349445730224336356.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca2349445730224336356.p12 thread started 2025-01-12T17:46:38,897 DEBUG [RPCClient-NioEventLoopGroup-3-11 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:38,915 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:38,929 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:38,930 DEBUG [FileChangeWatcher-hbase_test_key13496763275007779709.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key13496763275007779709.p12 thread started 2025-01-12T17:46:38,930 DEBUG [FileChangeWatcher-hbase_test_ca2349445730224336356.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca2349445730224336356.p12 thread started 2025-01-12T17:46:38,930 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xd0aea82a, L:/127.0.0.1:45497 - R:/127.0.0.1:59378] 2025-01-12T17:46:38,975 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59378, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:38,976 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:38,977 DEBUG [FileChangeWatcher-hbase_test_key13496763275007779709.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,977 DEBUG [FileChangeWatcher-hbase_test_ca2349445730224336356.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:38,977 DEBUG [FileChangeWatcher-hbase_test_key13496763275007779709.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key13496763275007779709.p12 thread finished 2025-01-12T17:46:38,977 DEBUG [FileChangeWatcher-hbase_test_ca2349445730224336356.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca2349445730224336356.p12 thread finished 2025-01-12T17:46:39,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7817aa4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:39,032 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:39,033 DEBUG [RPCClient-NioEventLoopGroup-3-12 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:39,065 DEBUG [RPCClient-NioEventLoopGroup-3-12 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:39,066 DEBUG [FileChangeWatcher-hbase_test_key13496763275007779709.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key13496763275007779709.p12 thread started 2025-01-12T17:46:39,066 DEBUG [FileChangeWatcher-hbase_test_ca2349445730224336356.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca2349445730224336356.p12 thread started 2025-01-12T17:46:39,066 DEBUG [RPCClient-NioEventLoopGroup-3-12 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:39,067 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x10f24105, L:/127.0.0.1:45497 - R:/127.0.0.1:59390] 2025-01-12T17:46:39,087 WARN [RPCClient-NioEventLoopGroup-3-12 {}] ipc.NettyRpcConnection$2(410): Exception encountered while connecting to the server localhost:45497 javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more 2025-01-12T17:46:39,088 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:39,089 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:45497 2025-01-12T17:46:39,089 DEBUG [FileChangeWatcher-hbase_test_ca2349445730224336356.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:39,089 DEBUG [RPCClient-NioEventLoopGroup-3-12 {}] ipc.FailedServers(52): Added failed server with address localhost:45497 to list caused by javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed 2025-01-12T17:46:39,089 DEBUG [FileChangeWatcher-hbase_test_ca2349445730224336356.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca2349445730224336356.p12 thread finished 2025-01-12T17:46:39,089 DEBUG [FileChangeWatcher-hbase_test_key13496763275007779709.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:39,089 DEBUG [FileChangeWatcher-hbase_test_ca2349445730224336356.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:39,089 DEBUG [FileChangeWatcher-hbase_test_key13496763275007779709.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key13496763275007779709.p12 thread finished 2025-01-12T17:46:39,089 DEBUG [FileChangeWatcher-hbase_test_ca2349445730224336356.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca2349445730224336356.p12 thread finished 2025-01-12T17:46:39,089 DEBUG [FileChangeWatcher-hbase_test_key13496763275007779709.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:39,089 DEBUG [RPCClient-NioEventLoopGroup-3-12 {}] ipc.BufferCallBeforeInitHandler(140): got ssl exception, which should have already been proceeded, log it here to prevent it being passed to netty's TailContext and trigger a confusing WARN message org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more 2025-01-12T17:46:39,090 DEBUG [FileChangeWatcher-hbase_test_key13496763275007779709.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key13496763275007779709.p12 thread finished 2025-01-12T17:46:39,089 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(470): Could not get peer certificate on TLS connection from /127.0.0.1:59390, although one is required javax.net.ssl.SSLPeerUnverifiedException: peer not authenticated at sun.security.ssl.SSLSessionImpl.getPeerCertificates(SSLSessionImpl.java:1034) ~[?:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.sslHandshakeCompleteHandler(NettyRpcServer.java:456) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.lambda$initSSL$0(NettyRpcServer.java:446) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.handleUnwrapThrowable(SslHandler.java:1381) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1363) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2025-01-12T17:46:39,090 WARN [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServerPreambleHandler(100): Connection /127.0.0.1:59390; caught unexpected downstream exception. org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.Alert.createSSLException(Alert.java:117) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:365) ~[?:?] at sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:293) ~[?:?] at sun.security.ssl.TransportContext.dispatch(TransportContext.java:204) ~[?:?] at sun.security.ssl.SSLTransport.decode(SSLTransport.java:172) ~[?:?] at sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:736) ~[?:?] at sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:691) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:506) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:482) ~[?:?] at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:679) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:308) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1465) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 15 more 2025-01-12T17:46:39,099 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[6: keyType=EC, storeFileType=PKCS12] Thread=35 (was 35), OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=312 (was 312), ProcessCount=11 (was 11), AvailableMemoryMB=3849 (was 3875) 2025-01-12T17:46:39,107 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[7: keyType=EC, storeFileType=BCFKS] Thread=35, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=312, ProcessCount=11, AvailableMemoryMB=3847 2025-01-12T17:46:39,564 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-12T17:46:39,564 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:35555 2025-01-12T17:46:39,565 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-12T17:46:39,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f6e351f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:39,566 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:39,570 DEBUG [RPCClient-NioEventLoopGroup-3-13 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:39,910 DEBUG [RPCClient-NioEventLoopGroup-3-13 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:39,911 DEBUG [FileChangeWatcher-hbase_test_key2891488035576771615.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key2891488035576771615.bcfks thread started 2025-01-12T17:46:39,911 DEBUG [RPCClient-NioEventLoopGroup-3-13 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:39,911 DEBUG [FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks thread started 2025-01-12T17:46:40,142 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:40,355 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:40,356 DEBUG [FileChangeWatcher-hbase_test_key2891488035576771615.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key2891488035576771615.bcfks thread started 2025-01-12T17:46:40,356 DEBUG [FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks thread started 2025-01-12T17:46:40,356 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x9da706ad, L:/127.0.0.1:35555 - R:/127.0.0.1:56762] 2025-01-12T17:46:40,622 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56762, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:41,041 INFO [Time-limited test {}] ipc.AbstractRpcClient(478): The server on localhost,35555,0 is dead - stopping the connection localhost:35555/TestProtobufRpcProto/jenkins (auth:SIMPLE) 2025-01-12T17:46:41,041 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:41,042 DEBUG [RPCClient-NioEventLoopGroup-3-14 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:41,043 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xfe6ada84, L:/127.0.0.1:35555 - R:/127.0.0.1:56778] 2025-01-12T17:46:41,047 INFO [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56778, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:41,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:41,048 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:35555 2025-01-12T17:46:41,048 DEBUG [FileChangeWatcher-hbase_test_key2891488035576771615.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:41,048 DEBUG [FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:41,048 DEBUG [FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:41,048 DEBUG [FileChangeWatcher-hbase_test_key2891488035576771615.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key2891488035576771615.bcfks thread finished 2025-01-12T17:46:41,048 DEBUG [FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks thread finished 2025-01-12T17:46:41,048 DEBUG [FileChangeWatcher-hbase_test_key2891488035576771615.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:41,048 DEBUG [FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks thread finished 2025-01-12T17:46:41,049 DEBUG [FileChangeWatcher-hbase_test_key2891488035576771615.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key2891488035576771615.bcfks thread finished 2025-01-12T17:46:41,068 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[7: keyType=EC, storeFileType=BCFKS] Thread=35 (was 35), OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=312 (was 312), ProcessCount=11 (was 11), AvailableMemoryMB=3672 (was 3847) 2025-01-12T17:46:41,076 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[7: keyType=EC, storeFileType=BCFKS] Thread=35, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=312, ProcessCount=11, AvailableMemoryMB=3672 2025-01-12T17:46:41,078 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-12T17:46:41,079 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:44619 2025-01-12T17:46:41,079 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-12T17:46:41,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ef107f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:41,080 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:41,080 DEBUG [RPCClient-NioEventLoopGroup-3-15 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:41,442 DEBUG [RPCClient-NioEventLoopGroup-3-15 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:41,444 DEBUG [FileChangeWatcher-hbase_test_key2891488035576771615.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key2891488035576771615.bcfks thread started 2025-01-12T17:46:41,445 DEBUG [FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks thread started 2025-01-12T17:46:41,445 DEBUG [RPCClient-NioEventLoopGroup-3-15 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:41,672 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:41,841 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:41,842 DEBUG [FileChangeWatcher-hbase_test_key2891488035576771615.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key2891488035576771615.bcfks thread started 2025-01-12T17:46:41,842 DEBUG [FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks thread started 2025-01-12T17:46:41,842 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x1efefc7a, L:/127.0.0.1:44619 - R:/127.0.0.1:41616] 2025-01-12T17:46:42,008 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41616, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-12T17:46:42,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:42,011 DEBUG [FileChangeWatcher-hbase_test_key2891488035576771615.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:42,012 DEBUG [FileChangeWatcher-hbase_test_key2891488035576771615.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key2891488035576771615.bcfks thread finished 2025-01-12T17:46:42,012 DEBUG [FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:42,012 DEBUG [FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks thread finished 2025-01-12T17:46:42,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4902961a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-12T17:46:42,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-12T17:46:42,417 DEBUG [RPCClient-NioEventLoopGroup-3-16 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-12T17:46:42,744 DEBUG [RPCClient-NioEventLoopGroup-3-16 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-12T17:46:42,744 DEBUG [FileChangeWatcher-hbase_test_key2891488035576771615.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key2891488035576771615.bcfks thread started 2025-01-12T17:46:42,744 DEBUG [FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks thread started 2025-01-12T17:46:42,745 DEBUG [RPCClient-NioEventLoopGroup-3-16 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-12T17:46:42,745 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x7fd8483a, L:/127.0.0.1:44619 - R:/127.0.0.1:41624] 2025-01-12T17:46:42,752 WARN [RPCClient-NioEventLoopGroup-3-16 {}] ipc.NettyRpcConnection$2(410): Exception encountered while connecting to the server localhost:44619 javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more 2025-01-12T17:46:42,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-12T17:46:42,753 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:44619 2025-01-12T17:46:42,753 DEBUG [RPCClient-NioEventLoopGroup-3-16 {}] ipc.FailedServers(52): Added failed server with address localhost:44619 to list caused by javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed 2025-01-12T17:46:42,753 DEBUG [FileChangeWatcher-hbase_test_key2891488035576771615.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:42,753 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(470): Could not get peer certificate on TLS connection from /127.0.0.1:41624, although one is required javax.net.ssl.SSLPeerUnverifiedException: peer not authenticated at sun.security.ssl.SSLSessionImpl.getPeerCertificates(SSLSessionImpl.java:1034) ~[?:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.sslHandshakeCompleteHandler(NettyRpcServer.java:456) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.lambda$initSSL$0(NettyRpcServer.java:446) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.handleUnwrapThrowable(SslHandler.java:1381) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1363) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2025-01-12T17:46:42,753 DEBUG [FileChangeWatcher-hbase_test_key2891488035576771615.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key2891488035576771615.bcfks thread finished 2025-01-12T17:46:42,754 DEBUG [FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:42,754 DEBUG [FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks thread finished 2025-01-12T17:46:42,754 DEBUG [FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:42,753 DEBUG [RPCClient-NioEventLoopGroup-3-16 {}] ipc.BufferCallBeforeInitHandler(140): got ssl exception, which should have already been proceeded, log it here to prevent it being passed to netty's TailContext and trigger a confusing WARN message org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more 2025-01-12T17:46:42,754 DEBUG [FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca13382481434363306431.bcfks thread finished 2025-01-12T17:46:42,754 WARN [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServerPreambleHandler(100): Connection /127.0.0.1:41624; caught unexpected downstream exception. org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.Alert.createSSLException(Alert.java:117) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:365) ~[?:?] at sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:293) ~[?:?] at sun.security.ssl.TransportContext.dispatch(TransportContext.java:204) ~[?:?] at sun.security.ssl.SSLTransport.decode(SSLTransport.java:172) ~[?:?] at sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:736) ~[?:?] at sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:691) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:506) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:482) ~[?:?] at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:679) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:308) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1465) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 15 more 2025-01-12T17:46:42,754 DEBUG [FileChangeWatcher-hbase_test_key2891488035576771615.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-12T17:46:42,755 DEBUG [FileChangeWatcher-hbase_test_key2891488035576771615.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key2891488035576771615.bcfks thread finished 2025-01-12T17:46:42,766 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[7: keyType=EC, storeFileType=BCFKS] Thread=35 (was 35), OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=343 (was 312) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3523 (was 3672)