2025-01-10 07:54:03,863 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4f4c4b1a 2025-01-10 07:54:03,896 main DEBUG Took 0.024277 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2025-01-10 07:54:03,897 main DEBUG PluginManager 'Core' found 129 plugins 2025-01-10 07:54:03,897 main DEBUG PluginManager 'Level' found 0 plugins 2025-01-10 07:54:03,908 main DEBUG PluginManager 'Lookup' found 16 plugins 2025-01-10 07:54:03,915 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:54:03,944 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2025-01-10 07:54:03,963 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:54:03,965 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:54:03,966 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:54:03,966 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:54:03,967 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:54:03,967 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:54:03,968 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:54:03,969 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:54:03,969 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:54:03,970 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:54:03,975 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:54:03,975 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:54:03,976 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:54:03,976 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:54:03,977 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:54:03,977 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:54:03,978 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:54:03,979 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:54:03,979 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:54:03,980 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:54:03,980 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:54:03,981 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:54:03,981 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:54:03,982 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:54:03,982 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:54:03,983 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2025-01-10 07:54:03,984 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:54:03,988 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2025-01-10 07:54:03,991 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2025-01-10 07:54:03,991 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2025-01-10 07:54:03,993 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2025-01-10 07:54:03,993 main DEBUG PluginManager 'Converter' found 47 plugins 2025-01-10 07:54:04,008 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2025-01-10 07:54:04,016 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2025-01-10 07:54:04,019 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2025-01-10 07:54:04,021 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2025-01-10 07:54:04,022 main DEBUG createAppenders(={Console}) 2025-01-10 07:54:04,023 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4f4c4b1a initialized 2025-01-10 07:54:04,023 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4f4c4b1a 2025-01-10 07:54:04,025 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4f4c4b1a OK. 2025-01-10 07:54:04,026 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2025-01-10 07:54:04,028 main DEBUG OutputStream closed 2025-01-10 07:54:04,030 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2025-01-10 07:54:04,030 main DEBUG Appender DefaultConsole-1 stopped with status true 2025-01-10 07:54:04,031 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6b695b06 OK 2025-01-10 07:54:04,226 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2025-01-10 07:54:04,229 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2025-01-10 07:54:04,231 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2025-01-10 07:54:04,232 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2025-01-10 07:54:04,233 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2025-01-10 07:54:04,233 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2025-01-10 07:54:04,234 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2025-01-10 07:54:04,234 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2025-01-10 07:54:04,235 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2025-01-10 07:54:04,235 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2025-01-10 07:54:04,236 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2025-01-10 07:54:04,236 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2025-01-10 07:54:04,236 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2025-01-10 07:54:04,237 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2025-01-10 07:54:04,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2025-01-10 07:54:04,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2025-01-10 07:54:04,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2025-01-10 07:54:04,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2025-01-10 07:54:04,251 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2025-01-10 07:54:04,252 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@366ac49b) with optional ClassLoader: null 2025-01-10 07:54:04,252 main DEBUG Shutdown hook enabled. Registering a new one. 2025-01-10 07:54:04,253 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@366ac49b] started OK. 2025-01-10T07:54:04,298 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.security.TestNettyTLSIPCFileWatcher timeout: 104 mins 2025-01-10 07:54:04,301 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2025-01-10 07:54:04,302 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2025-01-10T07:54:05,273 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2025-01-10T07:54:06,743 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[0: keyType=RSA, storeFileType=JKS] Thread=12, OpenFileDescriptor=295, MaxFileDescriptor=1048576, SystemLoadAverage=597, ProcessCount=13, AvailableMemoryMB=4314 2025-01-10T07:54:07,919 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2025-01-10T07:54:08,117 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2025-01-10T07:54:08,139 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2025-01-10T07:54:08,159 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2025-01-10T07:54:08,368 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-10T07:54:08,453 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 17156 (auto-detected) 2025-01-10T07:54:08,457 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2025-01-10T07:54:08,492 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:38717 2025-01-10T07:54:08,496 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-10T07:54:08,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d411d24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:08,548 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2025-01-10T07:54:08,550 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2025-01-10T07:54:09,090 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:09,164 DEBUG [RPCClient-NioEventLoopGroup-3-1 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:09,297 DEBUG [RPCClient-NioEventLoopGroup-3-1 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:09,446 DEBUG [FileChangeWatcher-hbase_test_key625914023657191166.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key625914023657191166.jks thread started 2025-01-10T07:54:09,452 DEBUG [FileChangeWatcher-hbase_test_ca86327516704131267.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca86327516704131267.jks thread started 2025-01-10T07:54:09,456 DEBUG [RPCClient-NioEventLoopGroup-3-1 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:09,580 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:09,611 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:09,718 DEBUG [FileChangeWatcher-hbase_test_key625914023657191166.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key625914023657191166.jks thread started 2025-01-10T07:54:09,719 DEBUG [FileChangeWatcher-hbase_test_ca86327516704131267.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca86327516704131267.jks thread started 2025-01-10T07:54:09,720 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xeed1017f, L:/127.0.0.1:38717 - R:/127.0.0.1:60748] 2025-01-10T07:54:09,905 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60748, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:10,296 INFO [Time-limited test {}] ipc.AbstractRpcClient(478): The server on localhost,38717,0 is dead - stopping the connection localhost:38717/TestProtobufRpcProto/jenkins (auth:SIMPLE) 2025-01-10T07:54:10,297 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:10,300 DEBUG [RPCClient-NioEventLoopGroup-3-2 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:10,306 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x3ff3f43b, L:/127.0.0.1:38717 - R:/127.0.0.1:60758] 2025-01-10T07:54:10,325 INFO [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60758, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:10,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:10,329 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:38717 2025-01-10T07:54:10,333 DEBUG [FileChangeWatcher-hbase_test_ca86327516704131267.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:10,334 DEBUG [FileChangeWatcher-hbase_test_ca86327516704131267.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:10,334 DEBUG [FileChangeWatcher-hbase_test_key625914023657191166.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:10,334 DEBUG [FileChangeWatcher-hbase_test_ca86327516704131267.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca86327516704131267.jks thread finished 2025-01-10T07:54:10,334 DEBUG [FileChangeWatcher-hbase_test_ca86327516704131267.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca86327516704131267.jks thread finished 2025-01-10T07:54:10,334 DEBUG [FileChangeWatcher-hbase_test_key625914023657191166.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key625914023657191166.jks thread finished 2025-01-10T07:54:10,333 DEBUG [FileChangeWatcher-hbase_test_key625914023657191166.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:10,335 DEBUG [FileChangeWatcher-hbase_test_key625914023657191166.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key625914023657191166.jks thread finished 2025-01-10T07:54:10,347 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[0: keyType=RSA, storeFileType=JKS] Thread=21 (was 12) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestNettyTLSIPCFileWatcher-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestNettyTLSIPCFileWatcher-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: TestNettyTLSIPCFileWatcher-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=327 (was 295) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=653 (was 597) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 13), AvailableMemoryMB=4068 (was 4314) 2025-01-10T07:54:10,358 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[0: keyType=RSA, storeFileType=JKS] Thread=21, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=653, ProcessCount=11, AvailableMemoryMB=4067 2025-01-10T07:54:10,360 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-10T07:54:10,360 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:32981 2025-01-10T07:54:10,361 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-10T07:54:10,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c05dee1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:10,362 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:10,364 DEBUG [RPCClient-NioEventLoopGroup-3-3 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:10,422 DEBUG [RPCClient-NioEventLoopGroup-3-3 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:10,423 DEBUG [FileChangeWatcher-hbase_test_ca86327516704131267.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca86327516704131267.jks thread started 2025-01-10T07:54:10,424 DEBUG [RPCClient-NioEventLoopGroup-3-3 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:10,423 DEBUG [FileChangeWatcher-hbase_test_key625914023657191166.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key625914023657191166.jks thread started 2025-01-10T07:54:10,452 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:10,479 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:10,482 DEBUG [FileChangeWatcher-hbase_test_key625914023657191166.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key625914023657191166.jks thread started 2025-01-10T07:54:10,482 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x9e3dcea0, L:/127.0.0.1:32981 - R:/127.0.0.1:37692] 2025-01-10T07:54:10,483 DEBUG [FileChangeWatcher-hbase_test_ca86327516704131267.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca86327516704131267.jks thread started 2025-01-10T07:54:10,555 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37692, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:10,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:10,559 DEBUG [FileChangeWatcher-hbase_test_key625914023657191166.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:10,560 DEBUG [FileChangeWatcher-hbase_test_ca86327516704131267.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:10,564 DEBUG [FileChangeWatcher-hbase_test_key625914023657191166.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key625914023657191166.jks thread finished 2025-01-10T07:54:10,564 DEBUG [FileChangeWatcher-hbase_test_ca86327516704131267.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca86327516704131267.jks thread finished 2025-01-10T07:54:11,131 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@284f296c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:11,132 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:11,135 DEBUG [RPCClient-NioEventLoopGroup-3-4 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:11,193 DEBUG [RPCClient-NioEventLoopGroup-3-4 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:11,210 DEBUG [FileChangeWatcher-hbase_test_key625914023657191166.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key625914023657191166.jks thread started 2025-01-10T07:54:11,214 DEBUG [RPCClient-NioEventLoopGroup-3-4 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:11,214 DEBUG [FileChangeWatcher-hbase_test_ca86327516704131267.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca86327516704131267.jks thread started 2025-01-10T07:54:11,219 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x7c4fa9d9, L:/127.0.0.1:32981 - R:/127.0.0.1:55244] 2025-01-10T07:54:11,239 WARN [RPCClient-NioEventLoopGroup-3-4 {}] ipc.NettyRpcConnection$2(410): Exception encountered while connecting to the server localhost:32981 javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more 2025-01-10T07:54:11,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:11,241 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:32981 2025-01-10T07:54:11,242 DEBUG [FileChangeWatcher-hbase_test_ca86327516704131267.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:11,242 DEBUG [FileChangeWatcher-hbase_test_key625914023657191166.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:11,242 DEBUG [FileChangeWatcher-hbase_test_ca86327516704131267.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca86327516704131267.jks thread finished 2025-01-10T07:54:11,242 DEBUG [FileChangeWatcher-hbase_test_key625914023657191166.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key625914023657191166.jks thread finished 2025-01-10T07:54:11,242 DEBUG [RPCClient-NioEventLoopGroup-3-4 {}] ipc.FailedServers(52): Added failed server with address localhost:32981 to list caused by javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed 2025-01-10T07:54:11,243 DEBUG [RPCClient-NioEventLoopGroup-3-4 {}] ipc.BufferCallBeforeInitHandler(140): got ssl exception, which should have already been proceeded, log it here to prevent it being passed to netty's TailContext and trigger a confusing WARN message org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more 2025-01-10T07:54:11,246 DEBUG [FileChangeWatcher-hbase_test_key625914023657191166.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:11,246 DEBUG [FileChangeWatcher-hbase_test_ca86327516704131267.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:11,246 DEBUG [FileChangeWatcher-hbase_test_key625914023657191166.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key625914023657191166.jks thread finished 2025-01-10T07:54:11,246 DEBUG [FileChangeWatcher-hbase_test_ca86327516704131267.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca86327516704131267.jks thread finished 2025-01-10T07:54:11,247 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(470): Could not get peer certificate on TLS connection from /127.0.0.1:55244, although one is required javax.net.ssl.SSLPeerUnverifiedException: peer not authenticated at sun.security.ssl.SSLSessionImpl.getPeerCertificates(SSLSessionImpl.java:1034) ~[?:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.sslHandshakeCompleteHandler(NettyRpcServer.java:456) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.lambda$initSSL$0(NettyRpcServer.java:446) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.handleUnwrapThrowable(SslHandler.java:1381) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1363) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2025-01-10T07:54:11,249 WARN [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServerPreambleHandler(100): Connection /127.0.0.1:55244; caught unexpected downstream exception. org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.Alert.createSSLException(Alert.java:117) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:365) ~[?:?] at sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:293) ~[?:?] at sun.security.ssl.TransportContext.dispatch(TransportContext.java:204) ~[?:?] at sun.security.ssl.SSLTransport.decode(SSLTransport.java:172) ~[?:?] at sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:736) ~[?:?] at sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:691) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:506) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:482) ~[?:?] at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:679) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:308) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1465) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 15 more 2025-01-10T07:54:11,265 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[0: keyType=RSA, storeFileType=JKS] Thread=23 (was 21) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=653 (was 653), ProcessCount=11 (was 11), AvailableMemoryMB=3950 (was 4067) 2025-01-10T07:54:11,280 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[1: keyType=RSA, storeFileType=PEM] Thread=23, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=653, ProcessCount=11, AvailableMemoryMB=3949 2025-01-10T07:54:11,437 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-10T07:54:11,439 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:46103 2025-01-10T07:54:11,441 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-10T07:54:11,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65cb6473, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:11,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:11,456 DEBUG [RPCClient-NioEventLoopGroup-3-5 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:11,492 DEBUG [RPCClient-NioEventLoopGroup-3-5 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:11,494 DEBUG [FileChangeWatcher-hbase_test_key4017799683391593929.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key4017799683391593929.pem thread started 2025-01-10T07:54:11,510 DEBUG [RPCClient-NioEventLoopGroup-3-5 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:11,510 DEBUG [FileChangeWatcher-hbase_test_ca1110704704797607045.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca1110704704797607045.pem thread started 2025-01-10T07:54:11,551 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:11,552 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:11,554 DEBUG [FileChangeWatcher-hbase_test_key4017799683391593929.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key4017799683391593929.pem thread started 2025-01-10T07:54:11,567 DEBUG [FileChangeWatcher-hbase_test_ca1110704704797607045.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca1110704704797607045.pem thread started 2025-01-10T07:54:11,567 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x23972993, L:/127.0.0.1:46103 - R:/127.0.0.1:47910] 2025-01-10T07:54:11,595 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47910, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:12,437 INFO [Time-limited test {}] ipc.AbstractRpcClient(478): The server on localhost,46103,0 is dead - stopping the connection localhost:46103/TestProtobufRpcProto/jenkins (auth:SIMPLE) 2025-01-10T07:54:12,441 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:12,444 DEBUG [RPCClient-NioEventLoopGroup-3-6 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:12,445 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xc4758044, L:/127.0.0.1:46103 - R:/127.0.0.1:47922] 2025-01-10T07:54:12,464 INFO [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47922, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:12,472 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:12,472 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:46103 2025-01-10T07:54:12,473 DEBUG [FileChangeWatcher-hbase_test_key4017799683391593929.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:12,473 DEBUG [FileChangeWatcher-hbase_test_key4017799683391593929.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key4017799683391593929.pem thread finished 2025-01-10T07:54:12,472 DEBUG [FileChangeWatcher-hbase_test_ca1110704704797607045.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:12,475 DEBUG [FileChangeWatcher-hbase_test_ca1110704704797607045.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca1110704704797607045.pem thread finished 2025-01-10T07:54:12,474 DEBUG [FileChangeWatcher-hbase_test_key4017799683391593929.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:12,476 DEBUG [FileChangeWatcher-hbase_test_key4017799683391593929.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key4017799683391593929.pem thread finished 2025-01-10T07:54:12,474 DEBUG [FileChangeWatcher-hbase_test_ca1110704704797607045.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:12,478 DEBUG [FileChangeWatcher-hbase_test_ca1110704704797607045.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca1110704704797607045.pem thread finished 2025-01-10T07:54:12,493 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[1: keyType=RSA, storeFileType=PEM] Thread=28 (was 23) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: FileChangeWatcher-hbase_test_ca1110704704797607045.pem app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeBytes(OutputStreamManager.java:353) app//org.apache.logging.log4j.core.layout.TextEncoderHelper.writeEncodedText(TextEncoderHelper.java:96) app//org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeText(TextEncoderHelper.java:65) app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:68) app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:32) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:227) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2017) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1983) app//org.apache.logging.slf4j.Log4jLogger.debug(Log4jLogger.java:139) app//org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:248) app//org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) Potentially hanging thread: FileChangeWatcher-hbase_test_ca1110704704797607045.pem app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$13(StackLocator.java:98) app//org.apache.logging.log4j.util.StackLocator$$Lambda$123/0x00007f8cc415fb98.test(Unknown Source) java.base@17.0.11/java.util.stream.WhileOps$1Op$1OpSink.accept(WhileOps.java:376) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.tryAdvance(StackStreamFactory.java:595) java.base@17.0.11/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129) java.base@17.0.11/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527) java.base@17.0.11/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513) java.base@17.0.11/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499) java.base@17.0.11/java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150) java.base@17.0.11/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) java.base@17.0.11/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647) app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$15(StackLocator.java:100) app//org.apache.logging.log4j.util.StackLocator$$Lambda$122/0x00007f8cc415f950.apply(Unknown Source) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243) java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499) app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97) app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121) app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2034) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1899) app//org.apache.logging.slf4j.Log4jLogger.debug(Log4jLogger.java:124) app//org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:219) Potentially hanging thread: FileChangeWatcher-hbase_test_key4017799683391593929.pem java.base@17.0.11/java.lang.ref.Reference.refersTo0(Native Method) java.base@17.0.11/java.lang.ref.Reference.refersToImpl(Reference.java:375) java.base@17.0.11/java.lang.ref.Reference.refersTo(Reference.java:366) java.base@17.0.11/java.lang.ThreadLocal$ThreadLocalMap.getEntry(ThreadLocal.java:436) java.base@17.0.11/java.lang.ThreadLocal.get(ThreadLocal.java:165) app//org.apache.logging.log4j.core.config.AppenderControl.isRecursiveCall(AppenderControl.java:109) app//org.apache.logging.log4j.core.config.AppenderControl.shouldSkip(AppenderControl.java:93) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:86) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2017) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1983) app//org.apache.logging.slf4j.Log4jLogger.debug(Log4jLogger.java:139) app//org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:248) app//org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-6 app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocal.get(FastThreadLocal.java:166) app//org.apache.hbase.thirdparty.io.netty.channel.ChannelOutboundBuffer.nioBuffers(ChannelOutboundBuffer.java:439) app//org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doWrite(NioSocketChannel.java:398) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.flush0(AbstractChannel.java:929) app//org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.flush0(AbstractNioChannel.java:359) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.flush(AbstractChannel.java:893) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.flush(DefaultChannelPipeline.java:1319) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:935) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:921) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:907) app//org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.forceFlush(SslHandler.java:2265) app//org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.wrapAndFlush(SslHandler.java:824) app//org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.flush(SslHandler.java:801) app//org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.flush(SslHandler.java:2094) app//org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.closeOutboundAndChannel(SslHandler.java:2063) app//org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.close(SslHandler.java:752) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeClose(AbstractChannelHandlerContext.java:761) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.close(AbstractChannelHandlerContext.java:733) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.close(AbstractChannelHandlerContext.java:560) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.close(DefaultChannelPipeline.java:906) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel.close(AbstractChannel.java:243) app//org.apache.hadoop.hbase.ipc.NettyRpcConnection.shutdown0(NettyRpcConnection.java:139) app//org.apache.hadoop.hbase.ipc.NettyRpcConnection$$Lambda$249/0x00007f8cc45a1ab0.run(Unknown Source) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:566) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=328 (was 327) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=653 (was 653), ProcessCount=11 (was 11), AvailableMemoryMB=3840 (was 3949) 2025-01-10T07:54:12,510 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[1: keyType=RSA, storeFileType=PEM] Thread=25, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=653, ProcessCount=11, AvailableMemoryMB=3839 2025-01-10T07:54:12,512 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-10T07:54:12,518 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:43689 2025-01-10T07:54:12,518 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-10T07:54:12,519 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66a5a763, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:12,520 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:12,527 DEBUG [RPCClient-NioEventLoopGroup-3-7 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:12,553 DEBUG [RPCClient-NioEventLoopGroup-3-7 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:12,562 DEBUG [FileChangeWatcher-hbase_test_key4017799683391593929.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key4017799683391593929.pem thread started 2025-01-10T07:54:12,564 DEBUG [RPCClient-NioEventLoopGroup-3-7 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:12,565 DEBUG [FileChangeWatcher-hbase_test_ca1110704704797607045.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca1110704704797607045.pem thread started 2025-01-10T07:54:12,598 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:12,599 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:12,606 DEBUG [FileChangeWatcher-hbase_test_key4017799683391593929.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key4017799683391593929.pem thread started 2025-01-10T07:54:12,606 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x22f273db, L:/127.0.0.1:43689 - R:/127.0.0.1:60614] 2025-01-10T07:54:12,629 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60614, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:12,630 DEBUG [FileChangeWatcher-hbase_test_ca1110704704797607045.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca1110704704797607045.pem thread started 2025-01-10T07:54:12,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:12,654 DEBUG [FileChangeWatcher-hbase_test_key4017799683391593929.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:12,654 DEBUG [FileChangeWatcher-hbase_test_key4017799683391593929.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key4017799683391593929.pem thread finished 2025-01-10T07:54:12,657 DEBUG [FileChangeWatcher-hbase_test_ca1110704704797607045.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:12,657 DEBUG [FileChangeWatcher-hbase_test_ca1110704704797607045.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca1110704704797607045.pem thread finished 2025-01-10T07:54:13,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cce3f8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:13,003 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:13,005 DEBUG [RPCClient-NioEventLoopGroup-3-8 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:13,012 DEBUG [RPCClient-NioEventLoopGroup-3-8 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:13,013 DEBUG [FileChangeWatcher-hbase_test_key4017799683391593929.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key4017799683391593929.pem thread started 2025-01-10T07:54:13,013 DEBUG [RPCClient-NioEventLoopGroup-3-8 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:13,014 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x2e4525df, L:/127.0.0.1:43689 - R:/127.0.0.1:60622] 2025-01-10T07:54:13,016 DEBUG [FileChangeWatcher-hbase_test_ca1110704704797607045.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca1110704704797607045.pem thread started 2025-01-10T07:54:13,025 WARN [RPCClient-NioEventLoopGroup-3-8 {}] ipc.NettyRpcConnection$2(410): Exception encountered while connecting to the server localhost:43689 javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more 2025-01-10T07:54:13,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:13,027 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:43689 2025-01-10T07:54:13,027 DEBUG [FileChangeWatcher-hbase_test_ca1110704704797607045.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:13,027 DEBUG [FileChangeWatcher-hbase_test_ca1110704704797607045.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:13,027 DEBUG [FileChangeWatcher-hbase_test_ca1110704704797607045.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca1110704704797607045.pem thread finished 2025-01-10T07:54:13,028 DEBUG [FileChangeWatcher-hbase_test_ca1110704704797607045.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca1110704704797607045.pem thread finished 2025-01-10T07:54:13,028 DEBUG [FileChangeWatcher-hbase_test_key4017799683391593929.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:13,028 DEBUG [FileChangeWatcher-hbase_test_key4017799683391593929.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key4017799683391593929.pem thread finished 2025-01-10T07:54:13,030 DEBUG [FileChangeWatcher-hbase_test_key4017799683391593929.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:13,030 DEBUG [FileChangeWatcher-hbase_test_key4017799683391593929.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key4017799683391593929.pem thread finished 2025-01-10T07:54:13,039 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(470): Could not get peer certificate on TLS connection from /127.0.0.1:60622, although one is required javax.net.ssl.SSLPeerUnverifiedException: peer not authenticated at sun.security.ssl.SSLSessionImpl.getPeerCertificates(SSLSessionImpl.java:1034) ~[?:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.sslHandshakeCompleteHandler(NettyRpcServer.java:456) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.lambda$initSSL$0(NettyRpcServer.java:446) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.setHandshakeFailure(SslHandler.java:2005) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.channelInactive(SslHandler.java:1165) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:405) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2025-01-10T07:54:13,040 DEBUG [RPCClient-NioEventLoopGroup-3-8 {}] ipc.FailedServers(52): Added failed server with address localhost:43689 to list caused by javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed 2025-01-10T07:54:13,041 DEBUG [RPCClient-NioEventLoopGroup-3-8 {}] ipc.BufferCallBeforeInitHandler(140): got ssl exception, which should have already been proceeded, log it here to prevent it being passed to netty's TailContext and trigger a confusing WARN message org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more 2025-01-10T07:54:13,047 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[1: keyType=RSA, storeFileType=PEM] Thread=27 (was 25) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-8 app//org.apache.hadoop.hbase.ipc.BlockingRpcCallback.run(BlockingRpcCallback.java:49) app//org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) app//org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) app//org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) app//org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) app//org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) app//org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) app//org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:107) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) app//org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.userEventTriggered(ChannelInboundHandlerAdapter.java:117) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.userEventTriggered(ByteToMessageDecoder.java:388) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) app//org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:211) app//org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$1000(NettyRpcConnection.java:85) app//org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:415) app//org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.lambda$operationComplete$0(NettyRpcConnection.java:431) app//org.apache.hadoop.hbase.ipc.NettyRpcConnection$2$$Lambda$216/0x00007f8cc454fa28.operationComplete(Unknown Source) app//org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) app//org.apache.hadoop.hbase.util.NettyFutureUtils$$Lambda$215/0x00007f8cc454ec18.operationComplete(Unknown Source) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) app//org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.handleUnwrapThrowable(SslHandler.java:1381) app//org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1363) app//org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) app//org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=328 (was 327) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=653 (was 653), ProcessCount=11 (was 11), AvailableMemoryMB=3781 (was 3839) 2025-01-10T07:54:13,054 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[2: keyType=RSA, storeFileType=PKCS12] Thread=27, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=653, ProcessCount=11, AvailableMemoryMB=3781 2025-01-10T07:54:13,127 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-10T07:54:13,129 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:33239 2025-01-10T07:54:13,129 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-10T07:54:13,132 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e282abe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:13,132 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:13,136 DEBUG [RPCClient-NioEventLoopGroup-3-9 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:13,192 DEBUG [RPCClient-NioEventLoopGroup-3-9 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:13,198 DEBUG [FileChangeWatcher-hbase_test_key4548997716961964184.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key4548997716961964184.p12 thread started 2025-01-10T07:54:13,199 DEBUG [RPCClient-NioEventLoopGroup-3-9 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:13,202 DEBUG [FileChangeWatcher-hbase_test_ca11606786854864650290.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca11606786854864650290.p12 thread started 2025-01-10T07:54:13,230 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:13,257 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:13,259 DEBUG [FileChangeWatcher-hbase_test_key4548997716961964184.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key4548997716961964184.p12 thread started 2025-01-10T07:54:13,260 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x47108773, L:/127.0.0.1:33239 - R:/127.0.0.1:58816] 2025-01-10T07:54:13,261 DEBUG [FileChangeWatcher-hbase_test_ca11606786854864650290.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca11606786854864650290.p12 thread started 2025-01-10T07:54:13,333 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58816, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:13,678 INFO [Time-limited test {}] ipc.AbstractRpcClient(478): The server on localhost,33239,0 is dead - stopping the connection localhost:33239/TestProtobufRpcProto/jenkins (auth:SIMPLE) 2025-01-10T07:54:13,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:13,681 DEBUG [RPCClient-NioEventLoopGroup-3-10 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:13,683 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xff571619, L:/127.0.0.1:33239 - R:/127.0.0.1:58818] 2025-01-10T07:54:13,693 INFO [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58818, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:13,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:13,696 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:33239 2025-01-10T07:54:13,696 DEBUG [FileChangeWatcher-hbase_test_key4548997716961964184.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:13,697 DEBUG [FileChangeWatcher-hbase_test_key4548997716961964184.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key4548997716961964184.p12 thread finished 2025-01-10T07:54:13,697 DEBUG [FileChangeWatcher-hbase_test_key4548997716961964184.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:13,697 DEBUG [FileChangeWatcher-hbase_test_key4548997716961964184.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key4548997716961964184.p12 thread finished 2025-01-10T07:54:13,698 DEBUG [FileChangeWatcher-hbase_test_ca11606786854864650290.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:13,698 DEBUG [FileChangeWatcher-hbase_test_ca11606786854864650290.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:13,698 DEBUG [FileChangeWatcher-hbase_test_ca11606786854864650290.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca11606786854864650290.p12 thread finished 2025-01-10T07:54:13,698 DEBUG [FileChangeWatcher-hbase_test_ca11606786854864650290.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca11606786854864650290.p12 thread finished 2025-01-10T07:54:13,711 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[2: keyType=RSA, storeFileType=PKCS12] Thread=29 (was 27) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=653 (was 653), ProcessCount=11 (was 11), AvailableMemoryMB=3727 (was 3781) 2025-01-10T07:54:13,724 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[2: keyType=RSA, storeFileType=PKCS12] Thread=29, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=653, ProcessCount=11, AvailableMemoryMB=3726 2025-01-10T07:54:13,726 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-10T07:54:13,729 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:36447 2025-01-10T07:54:13,729 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-10T07:54:13,730 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c74b06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:13,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:13,735 DEBUG [RPCClient-NioEventLoopGroup-3-11 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:13,794 DEBUG [RPCClient-NioEventLoopGroup-3-11 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:13,795 DEBUG [FileChangeWatcher-hbase_test_key4548997716961964184.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key4548997716961964184.p12 thread started 2025-01-10T07:54:13,795 DEBUG [RPCClient-NioEventLoopGroup-3-11 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:13,798 DEBUG [FileChangeWatcher-hbase_test_ca11606786854864650290.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca11606786854864650290.p12 thread started 2025-01-10T07:54:13,824 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:13,852 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:13,853 DEBUG [FileChangeWatcher-hbase_test_key4548997716961964184.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key4548997716961964184.p12 thread started 2025-01-10T07:54:13,854 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x2ceef2cc, L:/127.0.0.1:36447 - R:/127.0.0.1:50224] 2025-01-10T07:54:13,854 DEBUG [FileChangeWatcher-hbase_test_ca11606786854864650290.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca11606786854864650290.p12 thread started 2025-01-10T07:54:13,933 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50224, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:13,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:13,938 DEBUG [FileChangeWatcher-hbase_test_key4548997716961964184.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:13,939 DEBUG [FileChangeWatcher-hbase_test_key4548997716961964184.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key4548997716961964184.p12 thread finished 2025-01-10T07:54:13,939 DEBUG [FileChangeWatcher-hbase_test_ca11606786854864650290.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:13,940 DEBUG [FileChangeWatcher-hbase_test_ca11606786854864650290.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca11606786854864650290.p12 thread finished 2025-01-10T07:54:14,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39a7454, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:14,222 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:14,224 DEBUG [RPCClient-NioEventLoopGroup-3-12 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:14,275 DEBUG [RPCClient-NioEventLoopGroup-3-12 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:14,276 DEBUG [FileChangeWatcher-hbase_test_key4548997716961964184.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key4548997716961964184.p12 thread started 2025-01-10T07:54:14,277 DEBUG [RPCClient-NioEventLoopGroup-3-12 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:14,278 DEBUG [FileChangeWatcher-hbase_test_ca11606786854864650290.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca11606786854864650290.p12 thread started 2025-01-10T07:54:14,278 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x85dd6bbe, L:/127.0.0.1:36447 - R:/127.0.0.1:50238] 2025-01-10T07:54:14,309 WARN [RPCClient-NioEventLoopGroup-3-12 {}] ipc.NettyRpcConnection$2(410): Exception encountered while connecting to the server localhost:36447 javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more 2025-01-10T07:54:14,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:14,311 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:36447 2025-01-10T07:54:14,311 DEBUG [FileChangeWatcher-hbase_test_ca11606786854864650290.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:14,312 DEBUG [FileChangeWatcher-hbase_test_ca11606786854864650290.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca11606786854864650290.p12 thread finished 2025-01-10T07:54:14,312 DEBUG [FileChangeWatcher-hbase_test_ca11606786854864650290.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:14,312 DEBUG [FileChangeWatcher-hbase_test_ca11606786854864650290.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca11606786854864650290.p12 thread finished 2025-01-10T07:54:14,312 DEBUG [RPCClient-NioEventLoopGroup-3-12 {}] ipc.FailedServers(52): Added failed server with address localhost:36447 to list caused by javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed 2025-01-10T07:54:14,312 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(470): Could not get peer certificate on TLS connection from /127.0.0.1:50238, although one is required javax.net.ssl.SSLPeerUnverifiedException: peer not authenticated at sun.security.ssl.SSLSessionImpl.getPeerCertificates(SSLSessionImpl.java:1034) ~[?:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.sslHandshakeCompleteHandler(NettyRpcServer.java:456) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.lambda$initSSL$0(NettyRpcServer.java:446) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.handleUnwrapThrowable(SslHandler.java:1381) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1363) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2025-01-10T07:54:14,312 DEBUG [FileChangeWatcher-hbase_test_key4548997716961964184.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:14,313 DEBUG [FileChangeWatcher-hbase_test_key4548997716961964184.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key4548997716961964184.p12 thread finished 2025-01-10T07:54:14,313 DEBUG [RPCClient-NioEventLoopGroup-3-12 {}] ipc.BufferCallBeforeInitHandler(140): got ssl exception, which should have already been proceeded, log it here to prevent it being passed to netty's TailContext and trigger a confusing WARN message org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more 2025-01-10T07:54:14,313 WARN [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServerPreambleHandler(100): Connection /127.0.0.1:50238; caught unexpected downstream exception. org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.Alert.createSSLException(Alert.java:117) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:365) ~[?:?] at sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:293) ~[?:?] at sun.security.ssl.TransportContext.dispatch(TransportContext.java:204) ~[?:?] at sun.security.ssl.SSLTransport.decode(SSLTransport.java:172) ~[?:?] at sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:736) ~[?:?] at sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:691) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:506) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:482) ~[?:?] at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:679) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:308) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1465) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 15 more 2025-01-10T07:54:14,314 DEBUG [FileChangeWatcher-hbase_test_key4548997716961964184.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:14,314 DEBUG [FileChangeWatcher-hbase_test_key4548997716961964184.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key4548997716961964184.p12 thread finished 2025-01-10T07:54:14,324 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[2: keyType=RSA, storeFileType=PKCS12] Thread=32 (was 29) Potentially hanging thread: FifoRpcScheduler.handler-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=653 (was 653), ProcessCount=11 (was 11), AvailableMemoryMB=3676 (was 3726) 2025-01-10T07:54:14,331 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[3: keyType=RSA, storeFileType=BCFKS] Thread=31, OpenFileDescriptor=329, MaxFileDescriptor=1048576, SystemLoadAverage=653, ProcessCount=11, AvailableMemoryMB=3675 2025-01-10T07:54:15,116 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-10T07:54:15,119 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:44179 2025-01-10T07:54:15,121 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-10T07:54:15,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1844a7c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:15,122 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:15,129 DEBUG [RPCClient-NioEventLoopGroup-3-13 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:15,511 DEBUG [RPCClient-NioEventLoopGroup-3-13 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:15,514 DEBUG [FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks thread started 2025-01-10T07:54:15,514 DEBUG [RPCClient-NioEventLoopGroup-3-13 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:15,514 DEBUG [FileChangeWatcher-hbase_test_key14363626529732817252.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key14363626529732817252.bcfks thread started 2025-01-10T07:54:15,770 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:16,012 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:16,013 DEBUG [FileChangeWatcher-hbase_test_key14363626529732817252.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key14363626529732817252.bcfks thread started 2025-01-10T07:54:16,013 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x72e84f46, L:/127.0.0.1:44179 - R:/127.0.0.1:52762] 2025-01-10T07:54:16,013 DEBUG [FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks thread started 2025-01-10T07:54:16,230 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:52762, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:17,035 INFO [Time-limited test {}] ipc.AbstractRpcClient(478): The server on localhost,44179,0 is dead - stopping the connection localhost:44179/TestProtobufRpcProto/jenkins (auth:SIMPLE) 2025-01-10T07:54:17,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:17,039 DEBUG [RPCClient-NioEventLoopGroup-3-14 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:17,040 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xd5949085, L:/127.0.0.1:44179 - R:/127.0.0.1:52772] 2025-01-10T07:54:17,048 INFO [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:52772, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:17,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:17,050 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:44179 2025-01-10T07:54:17,050 DEBUG [FileChangeWatcher-hbase_test_key14363626529732817252.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:17,051 DEBUG [FileChangeWatcher-hbase_test_key14363626529732817252.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key14363626529732817252.bcfks thread finished 2025-01-10T07:54:17,051 DEBUG [FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:17,053 DEBUG [FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks thread finished 2025-01-10T07:54:17,055 DEBUG [FileChangeWatcher-hbase_test_key14363626529732817252.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:17,056 DEBUG [FileChangeWatcher-hbase_test_key14363626529732817252.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key14363626529732817252.bcfks thread finished 2025-01-10T07:54:17,050 DEBUG [FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:17,058 DEBUG [FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks thread finished 2025-01-10T07:54:17,062 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[3: keyType=RSA, storeFileType=BCFKS] Thread=37 (was 31) Potentially hanging thread: FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks java.base@17.0.11/java.lang.StringBuilder.append(StringBuilder.java:246) app//org.apache.logging.log4j.core.pattern.SimpleLiteralPatternConverter$Space.format(SimpleLiteralPatternConverter.java:101) app//org.apache.logging.log4j.core.pattern.SimpleLiteralPatternConverter.format(SimpleLiteralPatternConverter.java:56) app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:384) app//org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:240) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:225) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2017) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1983) app//org.apache.logging.slf4j.Log4jLogger.debug(Log4jLogger.java:139) app//org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:248) app//org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-14 java.base@17.0.11/com.sun.crypto.provider.GaloisCounterMode.checkReInit(GaloisCounterMode.java:329) java.base@17.0.11/com.sun.crypto.provider.GaloisCounterMode$GCMEncrypt.doFinal(GaloisCounterMode.java:1126) java.base@17.0.11/com.sun.crypto.provider.GaloisCounterMode.engineDoFinal(GaloisCounterMode.java:406) java.base@17.0.11/javax.crypto.Cipher.doFinal(Cipher.java:2088) java.base@17.0.11/sun.security.ssl.SSLCipher$T12GcmWriteCipherGenerator$GcmWriteCipher.dispose(SSLCipher.java:1812) java.base@17.0.11/sun.security.ssl.OutputRecord.close(OutputRecord.java:287) java.base@17.0.11/sun.security.ssl.SSLEngineOutputRecord.close(SSLEngineOutputRecord.java:61) java.base@17.0.11/sun.security.ssl.SSLEngineOutputRecord$HandshakeFragment.acquireCiphertext(SSLEngineOutputRecord.java:568) java.base@17.0.11/sun.security.ssl.SSLEngineOutputRecord.acquireCiphertext(SSLEngineOutputRecord.java:355) java.base@17.0.11/sun.security.ssl.SSLEngineOutputRecord.encode(SSLEngineOutputRecord.java:207) java.base@17.0.11/sun.security.ssl.SSLEngineOutputRecord.encode(SSLEngineOutputRecord.java:189) java.base@17.0.11/sun.security.ssl.SSLEngineImpl.encode(SSLEngineImpl.java:305) java.base@17.0.11/sun.security.ssl.SSLEngineImpl.writeRecord(SSLEngineImpl.java:246) java.base@17.0.11/sun.security.ssl.SSLEngineImpl.wrap(SSLEngineImpl.java:146) java.base@17.0.11/sun.security.ssl.SSLEngineImpl.wrap(SSLEngineImpl.java:123) java.base@17.0.11/javax.net.ssl.SSLEngine.wrap(SSLEngine.java:564) app//org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.wrap(SslHandler.java:1129) app//org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.wrap(SslHandler.java:867) app//org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.wrapAndFlush(SslHandler.java:820) app//org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.flush(SslHandler.java:801) app//org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.flush(SslHandler.java:2094) app//org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.closeOutboundAndChannel(SslHandler.java:2063) app//org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.close(SslHandler.java:752) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeClose(AbstractChannelHandlerContext.java:761) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.close(AbstractChannelHandlerContext.java:733) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.close(AbstractChannelHandlerContext.java:560) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.close(DefaultChannelPipeline.java:906) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel.close(AbstractChannel.java:243) app//org.apache.hadoop.hbase.ipc.NettyRpcConnection.shutdown0(NettyRpcConnection.java:139) app//org.apache.hadoop.hbase.ipc.NettyRpcConnection$$Lambda$249/0x00007f8cc45a1ab0.run(Unknown Source) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:566) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: FileChangeWatcher-hbase_test_key14363626529732817252.bcfks java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) app//org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) Potentially hanging thread: FifoRpcScheduler.handler-pool-0 java.base@17.0.11/java.lang.Throwable.(Throwable.java:255) java.base@17.0.11/java.lang.Exception.(Exception.java:55) java.base@17.0.11/java.lang.InterruptedException.(InterruptedException.java:57) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1640) java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks app//org.apache.logging.log4j.core.pattern.ExtendedThrowablePatternConverter.format(ExtendedThrowablePatternConverter.java:74) app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:384) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:375) app//org.apache.logging.log4j.core.layout.PatternLayout.toSerializable(PatternLayout.java:216) app//org.apache.logging.log4j.core.layout.PatternLayout.toSerializable(PatternLayout.java:59) app//org.apache.logging.log4j.core.layout.AbstractStringLayout.toByteArray(AbstractStringLayout.java:294) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:145) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2017) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1983) app//org.apache.logging.slf4j.Log4jLogger.debug(Log4jLogger.java:139) app//org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:248) app//org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) - Thread LEAK? -, OpenFileDescriptor=328 (was 329), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=681 (was 653) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3562 (was 3675) 2025-01-10T07:54:17,072 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[3: keyType=RSA, storeFileType=BCFKS] Thread=33, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=681, ProcessCount=11, AvailableMemoryMB=3561 2025-01-10T07:54:17,073 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-10T07:54:17,074 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:34593 2025-01-10T07:54:17,075 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-10T07:54:17,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cdeeb1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:17,076 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:17,078 DEBUG [RPCClient-NioEventLoopGroup-3-15 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:17,449 DEBUG [RPCClient-NioEventLoopGroup-3-15 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:17,450 DEBUG [FileChangeWatcher-hbase_test_key14363626529732817252.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key14363626529732817252.bcfks thread started 2025-01-10T07:54:17,451 DEBUG [FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks thread started 2025-01-10T07:54:17,452 DEBUG [RPCClient-NioEventLoopGroup-3-15 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:17,642 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:17,793 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:17,793 DEBUG [FileChangeWatcher-hbase_test_key14363626529732817252.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key14363626529732817252.bcfks thread started 2025-01-10T07:54:17,794 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x04a90ec9, L:/127.0.0.1:34593 - R:/127.0.0.1:50458] 2025-01-10T07:54:17,794 DEBUG [FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks thread started 2025-01-10T07:54:18,036 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50458, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:18,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:18,042 DEBUG [FileChangeWatcher-hbase_test_key14363626529732817252.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:18,042 DEBUG [FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:18,042 DEBUG [FileChangeWatcher-hbase_test_key14363626529732817252.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key14363626529732817252.bcfks thread finished 2025-01-10T07:54:18,042 DEBUG [FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks thread finished 2025-01-10T07:54:18,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56b915ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:18,928 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:18,935 DEBUG [RPCClient-NioEventLoopGroup-3-16 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:19,265 DEBUG [RPCClient-NioEventLoopGroup-3-16 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:19,266 DEBUG [RPCClient-NioEventLoopGroup-3-16 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:19,267 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x23f6e890, L:/127.0.0.1:34593 - R:/127.0.0.1:50468] 2025-01-10T07:54:19,268 DEBUG [FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks thread started 2025-01-10T07:54:19,268 DEBUG [FileChangeWatcher-hbase_test_key14363626529732817252.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key14363626529732817252.bcfks thread started 2025-01-10T07:54:19,277 WARN [RPCClient-NioEventLoopGroup-3-16 {}] ipc.NettyRpcConnection$2(410): Exception encountered while connecting to the server localhost:34593 javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more 2025-01-10T07:54:19,278 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:19,278 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:34593 2025-01-10T07:54:19,278 DEBUG [RPCClient-NioEventLoopGroup-3-16 {}] ipc.FailedServers(52): Added failed server with address localhost:34593 to list caused by javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed 2025-01-10T07:54:19,279 DEBUG [FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:19,279 DEBUG [FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks thread finished 2025-01-10T07:54:19,279 DEBUG [FileChangeWatcher-hbase_test_key14363626529732817252.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:19,279 DEBUG [FileChangeWatcher-hbase_test_key14363626529732817252.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key14363626529732817252.bcfks thread finished 2025-01-10T07:54:19,279 DEBUG [RPCClient-NioEventLoopGroup-3-16 {}] ipc.BufferCallBeforeInitHandler(140): got ssl exception, which should have already been proceeded, log it here to prevent it being passed to netty's TailContext and trigger a confusing WARN message org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more 2025-01-10T07:54:19,280 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(470): Could not get peer certificate on TLS connection from /127.0.0.1:50468, although one is required javax.net.ssl.SSLPeerUnverifiedException: peer not authenticated at sun.security.ssl.SSLSessionImpl.getPeerCertificates(SSLSessionImpl.java:1034) ~[?:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.sslHandshakeCompleteHandler(NettyRpcServer.java:456) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.lambda$initSSL$0(NettyRpcServer.java:446) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.handleUnwrapThrowable(SslHandler.java:1381) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1363) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2025-01-10T07:54:19,281 WARN [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServerPreambleHandler(100): Connection /127.0.0.1:50468; caught unexpected downstream exception. org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.Alert.createSSLException(Alert.java:117) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:365) ~[?:?] at sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:293) ~[?:?] at sun.security.ssl.TransportContext.dispatch(TransportContext.java:204) ~[?:?] at sun.security.ssl.SSLTransport.decode(SSLTransport.java:172) ~[?:?] at sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:736) ~[?:?] at sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:691) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:506) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:482) ~[?:?] at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:679) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:308) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1465) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 15 more 2025-01-10T07:54:19,282 DEBUG [FileChangeWatcher-hbase_test_key14363626529732817252.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:19,282 DEBUG [FileChangeWatcher-hbase_test_key14363626529732817252.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key14363626529732817252.bcfks thread finished 2025-01-10T07:54:19,282 DEBUG [FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:19,283 DEBUG [FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca1234750142646327869.bcfks thread finished 2025-01-10T07:54:19,293 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[3: keyType=RSA, storeFileType=BCFKS] Thread=36 (was 33) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: FifoRpcScheduler.handler-pool-0 java.base@17.0.11/java.lang.Thread.currentThread(Native Method) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.isReleasable(AbstractQueuedSynchronizer.java:502) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-3-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=681 (was 681), ProcessCount=11 (was 11), AvailableMemoryMB=3498 (was 3561) 2025-01-10T07:54:19,302 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[4: keyType=EC, storeFileType=JKS] Thread=35, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=681, ProcessCount=11, AvailableMemoryMB=3498 2025-01-10T07:54:19,489 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-10T07:54:19,490 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:37021 2025-01-10T07:54:19,490 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-10T07:54:19,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10969e67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:19,491 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:19,492 DEBUG [RPCClient-NioEventLoopGroup-3-1 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:19,525 DEBUG [RPCClient-NioEventLoopGroup-3-1 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:19,530 DEBUG [FileChangeWatcher-hbase_test_key13720493571086402012.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key13720493571086402012.jks thread started 2025-01-10T07:54:19,534 DEBUG [RPCClient-NioEventLoopGroup-3-1 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:19,535 DEBUG [FileChangeWatcher-hbase_test_ca11611007943733399680.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca11611007943733399680.jks thread started 2025-01-10T07:54:19,559 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:19,587 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:19,589 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x4df2dd69, L:/127.0.0.1:37021 - R:/127.0.0.1:33954] 2025-01-10T07:54:19,589 DEBUG [FileChangeWatcher-hbase_test_ca11611007943733399680.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca11611007943733399680.jks thread started 2025-01-10T07:54:19,596 DEBUG [FileChangeWatcher-hbase_test_key13720493571086402012.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key13720493571086402012.jks thread started 2025-01-10T07:54:19,664 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33954, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:19,720 INFO [Time-limited test {}] ipc.AbstractRpcClient(478): The server on localhost,37021,0 is dead - stopping the connection localhost:37021/TestProtobufRpcProto/jenkins (auth:SIMPLE) 2025-01-10T07:54:19,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:19,722 DEBUG [RPCClient-NioEventLoopGroup-3-2 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:19,723 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x9d563b85, L:/127.0.0.1:37021 - R:/127.0.0.1:33960] 2025-01-10T07:54:19,729 INFO [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33960, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:19,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:19,731 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:37021 2025-01-10T07:54:19,731 DEBUG [FileChangeWatcher-hbase_test_key13720493571086402012.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:19,731 DEBUG [FileChangeWatcher-hbase_test_key13720493571086402012.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:19,731 DEBUG [FileChangeWatcher-hbase_test_key13720493571086402012.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key13720493571086402012.jks thread finished 2025-01-10T07:54:19,731 DEBUG [FileChangeWatcher-hbase_test_key13720493571086402012.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key13720493571086402012.jks thread finished 2025-01-10T07:54:19,732 DEBUG [FileChangeWatcher-hbase_test_ca11611007943733399680.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:19,732 DEBUG [FileChangeWatcher-hbase_test_ca11611007943733399680.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca11611007943733399680.jks thread finished 2025-01-10T07:54:19,732 DEBUG [FileChangeWatcher-hbase_test_ca11611007943733399680.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:19,733 DEBUG [FileChangeWatcher-hbase_test_ca11611007943733399680.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca11611007943733399680.jks thread finished 2025-01-10T07:54:19,742 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[4: keyType=EC, storeFileType=JKS] Thread=35 (was 35), OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=681 (was 681), ProcessCount=11 (was 11), AvailableMemoryMB=3477 (was 3498) 2025-01-10T07:54:19,752 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[4: keyType=EC, storeFileType=JKS] Thread=35, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=681, ProcessCount=11, AvailableMemoryMB=3476 2025-01-10T07:54:19,753 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-10T07:54:19,756 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:34779 2025-01-10T07:54:19,756 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-10T07:54:19,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cc09ffe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:19,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:19,760 DEBUG [RPCClient-NioEventLoopGroup-3-3 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:19,815 DEBUG [RPCClient-NioEventLoopGroup-3-3 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:19,816 DEBUG [FileChangeWatcher-hbase_test_key13720493571086402012.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key13720493571086402012.jks thread started 2025-01-10T07:54:19,817 DEBUG [FileChangeWatcher-hbase_test_ca11611007943733399680.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca11611007943733399680.jks thread started 2025-01-10T07:54:19,817 DEBUG [RPCClient-NioEventLoopGroup-3-3 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:19,844 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:19,871 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:19,874 DEBUG [FileChangeWatcher-hbase_test_key13720493571086402012.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key13720493571086402012.jks thread started 2025-01-10T07:54:19,875 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x69d278b2, L:/127.0.0.1:34779 - R:/127.0.0.1:48948] 2025-01-10T07:54:19,876 DEBUG [FileChangeWatcher-hbase_test_ca11611007943733399680.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca11611007943733399680.jks thread started 2025-01-10T07:54:19,985 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48948, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:19,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:19,990 DEBUG [FileChangeWatcher-hbase_test_ca11611007943733399680.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:19,990 DEBUG [FileChangeWatcher-hbase_test_ca11611007943733399680.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca11611007943733399680.jks thread finished 2025-01-10T07:54:19,994 DEBUG [FileChangeWatcher-hbase_test_key13720493571086402012.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:19,994 DEBUG [FileChangeWatcher-hbase_test_key13720493571086402012.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key13720493571086402012.jks thread finished 2025-01-10T07:54:20,071 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48700e6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:20,072 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:20,073 DEBUG [RPCClient-NioEventLoopGroup-3-4 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:20,128 DEBUG [RPCClient-NioEventLoopGroup-3-4 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:20,135 DEBUG [RPCClient-NioEventLoopGroup-3-4 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:20,138 DEBUG [FileChangeWatcher-hbase_test_ca11611007943733399680.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca11611007943733399680.jks thread started 2025-01-10T07:54:20,138 DEBUG [FileChangeWatcher-hbase_test_key13720493571086402012.jks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key13720493571086402012.jks thread started 2025-01-10T07:54:20,141 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x7c9e2efc, L:/127.0.0.1:34779 - R:/127.0.0.1:48952] 2025-01-10T07:54:20,181 WARN [RPCClient-NioEventLoopGroup-3-4 {}] ipc.NettyRpcConnection$2(410): Exception encountered while connecting to the server localhost:34779 javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more 2025-01-10T07:54:20,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:20,184 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:34779 2025-01-10T07:54:20,184 DEBUG [RPCClient-NioEventLoopGroup-3-4 {}] ipc.FailedServers(52): Added failed server with address localhost:34779 to list caused by javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed 2025-01-10T07:54:20,184 DEBUG [FileChangeWatcher-hbase_test_key13720493571086402012.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:20,184 DEBUG [FileChangeWatcher-hbase_test_key13720493571086402012.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key13720493571086402012.jks thread finished 2025-01-10T07:54:20,184 DEBUG [FileChangeWatcher-hbase_test_ca11611007943733399680.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:20,184 DEBUG [FileChangeWatcher-hbase_test_ca11611007943733399680.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca11611007943733399680.jks thread finished 2025-01-10T07:54:20,184 DEBUG [FileChangeWatcher-hbase_test_ca11611007943733399680.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:20,184 DEBUG [FileChangeWatcher-hbase_test_ca11611007943733399680.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca11611007943733399680.jks thread finished 2025-01-10T07:54:20,184 DEBUG [FileChangeWatcher-hbase_test_key13720493571086402012.jks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:20,184 DEBUG [RPCClient-NioEventLoopGroup-3-4 {}] ipc.BufferCallBeforeInitHandler(140): got ssl exception, which should have already been proceeded, log it here to prevent it being passed to netty's TailContext and trigger a confusing WARN message org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more 2025-01-10T07:54:20,185 DEBUG [FileChangeWatcher-hbase_test_key13720493571086402012.jks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key13720493571086402012.jks thread finished 2025-01-10T07:54:20,199 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(470): Could not get peer certificate on TLS connection from /127.0.0.1:48952, although one is required javax.net.ssl.SSLPeerUnverifiedException: peer not authenticated at sun.security.ssl.SSLSessionImpl.getPeerCertificates(SSLSessionImpl.java:1034) ~[?:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.sslHandshakeCompleteHandler(NettyRpcServer.java:456) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.lambda$initSSL$0(NettyRpcServer.java:446) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.handleUnwrapThrowable(SslHandler.java:1381) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1363) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2025-01-10T07:54:20,201 WARN [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServerPreambleHandler(100): Connection /127.0.0.1:48952; caught unexpected downstream exception. org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.Alert.createSSLException(Alert.java:117) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:365) ~[?:?] at sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:293) ~[?:?] at sun.security.ssl.TransportContext.dispatch(TransportContext.java:204) ~[?:?] at sun.security.ssl.SSLTransport.decode(SSLTransport.java:172) ~[?:?] at sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:736) ~[?:?] at sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:691) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:506) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:482) ~[?:?] at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:679) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:308) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1465) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 15 more 2025-01-10T07:54:20,217 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[4: keyType=EC, storeFileType=JKS] Thread=35 (was 35), OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=681 (was 681), ProcessCount=11 (was 11), AvailableMemoryMB=3456 (was 3476) 2025-01-10T07:54:20,232 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[5: keyType=EC, storeFileType=PEM] Thread=35, OpenFileDescriptor=329, MaxFileDescriptor=1048576, SystemLoadAverage=681, ProcessCount=11, AvailableMemoryMB=3456 2025-01-10T07:54:20,240 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-10T07:54:20,242 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:32799 2025-01-10T07:54:20,242 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-10T07:54:20,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f0c2c74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:20,243 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:20,244 DEBUG [RPCClient-NioEventLoopGroup-3-5 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:20,252 DEBUG [RPCClient-NioEventLoopGroup-3-5 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:20,254 DEBUG [FileChangeWatcher-hbase_test_key17306192276133909092.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key17306192276133909092.pem thread started 2025-01-10T07:54:20,255 DEBUG [RPCClient-NioEventLoopGroup-3-5 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:20,257 DEBUG [FileChangeWatcher-hbase_test_ca989041532508332371.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca989041532508332371.pem thread started 2025-01-10T07:54:20,261 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:20,262 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:20,270 DEBUG [FileChangeWatcher-hbase_test_key17306192276133909092.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key17306192276133909092.pem thread started 2025-01-10T07:54:20,272 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xd18910d6, L:/127.0.0.1:32799 - R:/127.0.0.1:53198] 2025-01-10T07:54:20,276 DEBUG [FileChangeWatcher-hbase_test_ca989041532508332371.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca989041532508332371.pem thread started 2025-01-10T07:54:20,303 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53198, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:20,328 INFO [Time-limited test {}] ipc.AbstractRpcClient(478): The server on localhost,32799,0 is dead - stopping the connection localhost:32799/TestProtobufRpcProto/jenkins (auth:SIMPLE) 2025-01-10T07:54:20,331 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:20,339 DEBUG [RPCClient-NioEventLoopGroup-3-6 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:20,340 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x784ca263, L:/127.0.0.1:32799 - R:/127.0.0.1:53202] 2025-01-10T07:54:20,345 INFO [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53202, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:20,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:20,349 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:32799 2025-01-10T07:54:20,349 DEBUG [FileChangeWatcher-hbase_test_key17306192276133909092.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:20,349 DEBUG [FileChangeWatcher-hbase_test_key17306192276133909092.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key17306192276133909092.pem thread finished 2025-01-10T07:54:20,349 DEBUG [FileChangeWatcher-hbase_test_ca989041532508332371.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:20,349 DEBUG [FileChangeWatcher-hbase_test_key17306192276133909092.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:20,350 DEBUG [FileChangeWatcher-hbase_test_ca989041532508332371.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca989041532508332371.pem thread finished 2025-01-10T07:54:20,350 DEBUG [FileChangeWatcher-hbase_test_ca989041532508332371.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:20,350 DEBUG [FileChangeWatcher-hbase_test_key17306192276133909092.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key17306192276133909092.pem thread finished 2025-01-10T07:54:20,350 DEBUG [FileChangeWatcher-hbase_test_ca989041532508332371.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca989041532508332371.pem thread finished 2025-01-10T07:54:20,365 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[5: keyType=EC, storeFileType=PEM] Thread=35 (was 35), OpenFileDescriptor=327 (was 329), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=795 (was 681) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3450 (was 3456) 2025-01-10T07:54:20,377 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[5: keyType=EC, storeFileType=PEM] Thread=35, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=795, ProcessCount=11, AvailableMemoryMB=3447 2025-01-10T07:54:20,379 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-10T07:54:20,380 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:45763 2025-01-10T07:54:20,380 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-10T07:54:20,381 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43d25a4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:20,382 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:20,382 DEBUG [RPCClient-NioEventLoopGroup-3-7 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:20,388 DEBUG [RPCClient-NioEventLoopGroup-3-7 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:20,394 DEBUG [FileChangeWatcher-hbase_test_key17306192276133909092.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key17306192276133909092.pem thread started 2025-01-10T07:54:20,394 DEBUG [RPCClient-NioEventLoopGroup-3-7 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:20,395 DEBUG [FileChangeWatcher-hbase_test_ca989041532508332371.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca989041532508332371.pem thread started 2025-01-10T07:54:20,403 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:20,403 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:20,406 DEBUG [FileChangeWatcher-hbase_test_key17306192276133909092.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key17306192276133909092.pem thread started 2025-01-10T07:54:20,407 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xb4372c62, L:/127.0.0.1:45763 - R:/127.0.0.1:40658] 2025-01-10T07:54:20,428 DEBUG [FileChangeWatcher-hbase_test_ca989041532508332371.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca989041532508332371.pem thread started 2025-01-10T07:54:20,441 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40658, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:20,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:20,454 DEBUG [FileChangeWatcher-hbase_test_key17306192276133909092.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:20,454 DEBUG [FileChangeWatcher-hbase_test_key17306192276133909092.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key17306192276133909092.pem thread finished 2025-01-10T07:54:20,470 DEBUG [FileChangeWatcher-hbase_test_ca989041532508332371.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:20,471 DEBUG [FileChangeWatcher-hbase_test_ca989041532508332371.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca989041532508332371.pem thread finished 2025-01-10T07:54:20,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa49f39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:20,479 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:20,480 DEBUG [RPCClient-NioEventLoopGroup-3-8 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:20,486 DEBUG [RPCClient-NioEventLoopGroup-3-8 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:20,498 DEBUG [FileChangeWatcher-hbase_test_key17306192276133909092.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key17306192276133909092.pem thread started 2025-01-10T07:54:20,502 DEBUG [FileChangeWatcher-hbase_test_ca989041532508332371.pem {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca989041532508332371.pem thread started 2025-01-10T07:54:20,502 DEBUG [RPCClient-NioEventLoopGroup-3-8 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:20,503 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x8b632564, L:/127.0.0.1:45763 - R:/127.0.0.1:35902] 2025-01-10T07:54:20,510 WARN [RPCClient-NioEventLoopGroup-3-8 {}] ipc.NettyRpcConnection$2(410): Exception encountered while connecting to the server localhost:45763 javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more 2025-01-10T07:54:20,511 DEBUG [RPCClient-NioEventLoopGroup-3-8 {}] ipc.FailedServers(52): Added failed server with address localhost:45763 to list caused by javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed 2025-01-10T07:54:20,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:20,511 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:45763 2025-01-10T07:54:20,512 DEBUG [FileChangeWatcher-hbase_test_key17306192276133909092.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:20,512 DEBUG [FileChangeWatcher-hbase_test_key17306192276133909092.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key17306192276133909092.pem thread finished 2025-01-10T07:54:20,512 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(470): Could not get peer certificate on TLS connection from /127.0.0.1:35902, although one is required javax.net.ssl.SSLPeerUnverifiedException: peer not authenticated at sun.security.ssl.SSLSessionImpl.getPeerCertificates(SSLSessionImpl.java:1034) ~[?:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.sslHandshakeCompleteHandler(NettyRpcServer.java:456) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.lambda$initSSL$0(NettyRpcServer.java:446) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.handleUnwrapThrowable(SslHandler.java:1381) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1363) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2025-01-10T07:54:20,512 DEBUG [FileChangeWatcher-hbase_test_ca989041532508332371.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:20,512 DEBUG [FileChangeWatcher-hbase_test_ca989041532508332371.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca989041532508332371.pem thread finished 2025-01-10T07:54:20,512 DEBUG [FileChangeWatcher-hbase_test_ca989041532508332371.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:20,514 DEBUG [FileChangeWatcher-hbase_test_ca989041532508332371.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca989041532508332371.pem thread finished 2025-01-10T07:54:20,512 WARN [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServerPreambleHandler(100): Connection /127.0.0.1:35902; caught unexpected downstream exception. org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.Alert.createSSLException(Alert.java:117) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:365) ~[?:?] at sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:293) ~[?:?] at sun.security.ssl.TransportContext.dispatch(TransportContext.java:204) ~[?:?] at sun.security.ssl.SSLTransport.decode(SSLTransport.java:172) ~[?:?] at sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:736) ~[?:?] at sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:691) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:506) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:482) ~[?:?] at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:679) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:308) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1465) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 15 more 2025-01-10T07:54:20,513 DEBUG [FileChangeWatcher-hbase_test_key17306192276133909092.pem {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:20,514 DEBUG [FileChangeWatcher-hbase_test_key17306192276133909092.pem {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key17306192276133909092.pem thread finished 2025-01-10T07:54:20,511 DEBUG [RPCClient-NioEventLoopGroup-3-8 {}] ipc.BufferCallBeforeInitHandler(140): got ssl exception, which should have already been proceeded, log it here to prevent it being passed to netty's TailContext and trigger a confusing WARN message org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more 2025-01-10T07:54:20,539 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[5: keyType=EC, storeFileType=PEM] Thread=35 (was 35), OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=795 (was 795), ProcessCount=11 (was 11), AvailableMemoryMB=3441 (was 3447) 2025-01-10T07:54:20,558 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[6: keyType=EC, storeFileType=PKCS12] Thread=35, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=795, ProcessCount=11, AvailableMemoryMB=3441 2025-01-10T07:54:20,612 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-10T07:54:20,613 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:45035 2025-01-10T07:54:20,613 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-10T07:54:20,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24594a4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:20,614 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:20,614 DEBUG [RPCClient-NioEventLoopGroup-3-9 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:20,644 DEBUG [RPCClient-NioEventLoopGroup-3-9 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:20,645 DEBUG [FileChangeWatcher-hbase_test_key7500338239844379813.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key7500338239844379813.p12 thread started 2025-01-10T07:54:20,645 DEBUG [FileChangeWatcher-hbase_test_ca18056987742412090464.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca18056987742412090464.p12 thread started 2025-01-10T07:54:20,645 DEBUG [RPCClient-NioEventLoopGroup-3-9 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:20,660 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:20,685 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:20,686 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x8174a5b5, L:/127.0.0.1:45035 - R:/127.0.0.1:60072] 2025-01-10T07:54:20,686 DEBUG [FileChangeWatcher-hbase_test_key7500338239844379813.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key7500338239844379813.p12 thread started 2025-01-10T07:54:20,687 DEBUG [FileChangeWatcher-hbase_test_ca18056987742412090464.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca18056987742412090464.p12 thread started 2025-01-10T07:54:20,723 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60072, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:20,773 INFO [Time-limited test {}] ipc.AbstractRpcClient(478): The server on localhost,45035,0 is dead - stopping the connection localhost:45035/TestProtobufRpcProto/jenkins (auth:SIMPLE) 2025-01-10T07:54:20,773 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:20,774 DEBUG [RPCClient-NioEventLoopGroup-3-10 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:20,775 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x758aaac7, L:/127.0.0.1:45035 - R:/127.0.0.1:60082] 2025-01-10T07:54:20,781 INFO [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60082, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:20,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:20,783 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:45035 2025-01-10T07:54:20,783 DEBUG [FileChangeWatcher-hbase_test_key7500338239844379813.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:20,784 DEBUG [FileChangeWatcher-hbase_test_ca18056987742412090464.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:20,784 DEBUG [FileChangeWatcher-hbase_test_ca18056987742412090464.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:20,784 DEBUG [FileChangeWatcher-hbase_test_ca18056987742412090464.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca18056987742412090464.p12 thread finished 2025-01-10T07:54:20,784 DEBUG [FileChangeWatcher-hbase_test_ca18056987742412090464.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca18056987742412090464.p12 thread finished 2025-01-10T07:54:20,784 DEBUG [FileChangeWatcher-hbase_test_key7500338239844379813.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key7500338239844379813.p12 thread finished 2025-01-10T07:54:20,784 DEBUG [FileChangeWatcher-hbase_test_key7500338239844379813.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:20,784 DEBUG [FileChangeWatcher-hbase_test_key7500338239844379813.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key7500338239844379813.p12 thread finished 2025-01-10T07:54:20,791 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[6: keyType=EC, storeFileType=PKCS12] Thread=35 (was 35), OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=795 (was 795), ProcessCount=11 (was 11), AvailableMemoryMB=3430 (was 3441) 2025-01-10T07:54:20,798 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[6: keyType=EC, storeFileType=PKCS12] Thread=35, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=795, ProcessCount=11, AvailableMemoryMB=3430 2025-01-10T07:54:20,798 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-10T07:54:20,799 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:44669 2025-01-10T07:54:20,799 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-10T07:54:20,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@330ff9ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:20,800 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:20,800 DEBUG [RPCClient-NioEventLoopGroup-3-11 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:20,832 DEBUG [RPCClient-NioEventLoopGroup-3-11 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:20,833 DEBUG [FileChangeWatcher-hbase_test_key7500338239844379813.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key7500338239844379813.p12 thread started 2025-01-10T07:54:20,833 DEBUG [FileChangeWatcher-hbase_test_ca18056987742412090464.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca18056987742412090464.p12 thread started 2025-01-10T07:54:20,833 DEBUG [RPCClient-NioEventLoopGroup-3-11 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:20,850 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:20,865 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:20,866 DEBUG [FileChangeWatcher-hbase_test_key7500338239844379813.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key7500338239844379813.p12 thread started 2025-01-10T07:54:20,866 DEBUG [FileChangeWatcher-hbase_test_ca18056987742412090464.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca18056987742412090464.p12 thread started 2025-01-10T07:54:20,866 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x8088e432, L:/127.0.0.1:44669 - R:/127.0.0.1:56756] 2025-01-10T07:54:20,922 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56756, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:20,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:20,924 DEBUG [FileChangeWatcher-hbase_test_key7500338239844379813.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:20,924 DEBUG [FileChangeWatcher-hbase_test_ca18056987742412090464.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:20,924 DEBUG [FileChangeWatcher-hbase_test_key7500338239844379813.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key7500338239844379813.p12 thread finished 2025-01-10T07:54:20,925 DEBUG [FileChangeWatcher-hbase_test_ca18056987742412090464.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca18056987742412090464.p12 thread finished 2025-01-10T07:54:20,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61bdf13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:20,982 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:20,982 DEBUG [RPCClient-NioEventLoopGroup-3-12 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:21,013 DEBUG [RPCClient-NioEventLoopGroup-3-12 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:21,014 DEBUG [FileChangeWatcher-hbase_test_key7500338239844379813.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key7500338239844379813.p12 thread started 2025-01-10T07:54:21,015 DEBUG [RPCClient-NioEventLoopGroup-3-12 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:21,016 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xc04d8652, L:/127.0.0.1:44669 - R:/127.0.0.1:56762] 2025-01-10T07:54:21,016 DEBUG [FileChangeWatcher-hbase_test_ca18056987742412090464.p12 {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca18056987742412090464.p12 thread started 2025-01-10T07:54:21,046 WARN [RPCClient-NioEventLoopGroup-3-12 {}] ipc.NettyRpcConnection$2(410): Exception encountered while connecting to the server localhost:44669 javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more 2025-01-10T07:54:21,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:21,047 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:44669 2025-01-10T07:54:21,047 DEBUG [FileChangeWatcher-hbase_test_key7500338239844379813.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:21,047 DEBUG [FileChangeWatcher-hbase_test_key7500338239844379813.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key7500338239844379813.p12 thread finished 2025-01-10T07:54:21,047 DEBUG [FileChangeWatcher-hbase_test_ca18056987742412090464.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:21,047 DEBUG [FileChangeWatcher-hbase_test_ca18056987742412090464.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca18056987742412090464.p12 thread finished 2025-01-10T07:54:21,048 DEBUG [RPCClient-NioEventLoopGroup-3-12 {}] ipc.FailedServers(52): Added failed server with address localhost:44669 to list caused by javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed 2025-01-10T07:54:21,048 DEBUG [FileChangeWatcher-hbase_test_key7500338239844379813.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:21,048 DEBUG [FileChangeWatcher-hbase_test_key7500338239844379813.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key7500338239844379813.p12 thread finished 2025-01-10T07:54:21,048 DEBUG [RPCClient-NioEventLoopGroup-3-12 {}] ipc.BufferCallBeforeInitHandler(140): got ssl exception, which should have already been proceeded, log it here to prevent it being passed to netty's TailContext and trigger a confusing WARN message org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more 2025-01-10T07:54:21,048 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(470): Could not get peer certificate on TLS connection from /127.0.0.1:56762, although one is required javax.net.ssl.SSLPeerUnverifiedException: peer not authenticated at sun.security.ssl.SSLSessionImpl.getPeerCertificates(SSLSessionImpl.java:1034) ~[?:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.sslHandshakeCompleteHandler(NettyRpcServer.java:456) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.lambda$initSSL$0(NettyRpcServer.java:446) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.handleUnwrapThrowable(SslHandler.java:1381) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1363) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2025-01-10T07:54:21,049 DEBUG [FileChangeWatcher-hbase_test_ca18056987742412090464.p12 {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:21,049 DEBUG [FileChangeWatcher-hbase_test_ca18056987742412090464.p12 {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca18056987742412090464.p12 thread finished 2025-01-10T07:54:21,049 WARN [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServerPreambleHandler(100): Connection /127.0.0.1:56762; caught unexpected downstream exception. org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.Alert.createSSLException(Alert.java:117) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:365) ~[?:?] at sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:293) ~[?:?] at sun.security.ssl.TransportContext.dispatch(TransportContext.java:204) ~[?:?] at sun.security.ssl.SSLTransport.decode(SSLTransport.java:172) ~[?:?] at sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:736) ~[?:?] at sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:691) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:506) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:482) ~[?:?] at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:679) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:308) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1465) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 15 more 2025-01-10T07:54:21,058 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[6: keyType=EC, storeFileType=PKCS12] Thread=35 (was 35), OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=795 (was 795), ProcessCount=11 (was 11), AvailableMemoryMB=3407 (was 3430) 2025-01-10T07:54:21,068 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[7: keyType=EC, storeFileType=BCFKS] Thread=35, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=795, ProcessCount=11, AvailableMemoryMB=3407 2025-01-10T07:54:21,592 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-10T07:54:21,593 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:37171 2025-01-10T07:54:21,593 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-10T07:54:21,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594814c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:21,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:21,594 DEBUG [RPCClient-NioEventLoopGroup-3-13 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:21,901 DEBUG [RPCClient-NioEventLoopGroup-3-13 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:21,902 DEBUG [FileChangeWatcher-hbase_test_key11389729334378219996.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key11389729334378219996.bcfks thread started 2025-01-10T07:54:21,902 DEBUG [RPCClient-NioEventLoopGroup-3-13 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:21,903 DEBUG [FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks thread started 2025-01-10T07:54:22,137 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:22,303 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:22,304 DEBUG [FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks thread started 2025-01-10T07:54:22,304 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x7911dd63, L:/127.0.0.1:37171 - R:/127.0.0.1:42220] 2025-01-10T07:54:22,304 DEBUG [FileChangeWatcher-hbase_test_key11389729334378219996.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key11389729334378219996.bcfks thread started 2025-01-10T07:54:22,519 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42220, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:22,986 INFO [Time-limited test {}] ipc.AbstractRpcClient(478): The server on localhost,37171,0 is dead - stopping the connection localhost:37171/TestProtobufRpcProto/jenkins (auth:SIMPLE) 2025-01-10T07:54:22,986 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:22,987 DEBUG [RPCClient-NioEventLoopGroup-3-14 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:22,988 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x84819e2c, L:/127.0.0.1:37171 - R:/127.0.0.1:42230] 2025-01-10T07:54:22,992 INFO [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42230, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:22,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:22,994 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:37171 2025-01-10T07:54:22,994 DEBUG [FileChangeWatcher-hbase_test_key11389729334378219996.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:22,994 DEBUG [FileChangeWatcher-hbase_test_key11389729334378219996.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:22,994 DEBUG [FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:22,994 DEBUG [FileChangeWatcher-hbase_test_key11389729334378219996.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key11389729334378219996.bcfks thread finished 2025-01-10T07:54:22,994 DEBUG [FileChangeWatcher-hbase_test_key11389729334378219996.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key11389729334378219996.bcfks thread finished 2025-01-10T07:54:22,994 DEBUG [FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks thread finished 2025-01-10T07:54:22,994 DEBUG [FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:22,994 DEBUG [FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks thread finished 2025-01-10T07:54:23,003 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceClientAndServerKeystore[7: keyType=EC, storeFileType=BCFKS] Thread=35 (was 35), OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=795 (was 795), ProcessCount=11 (was 11), AvailableMemoryMB=3387 (was 3407) 2025-01-10T07:54:23,011 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[7: keyType=EC, storeFileType=BCFKS] Thread=35, OpenFileDescriptor=327, MaxFileDescriptor=1048576, SystemLoadAverage=795, ProcessCount=11, AvailableMemoryMB=3386 2025-01-10T07:54:23,013 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-10T07:54:23,014 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /127.0.0.1:36011 2025-01-10T07:54:23,014 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2025-01-10T07:54:23,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c1f2297, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:23,015 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:23,016 DEBUG [RPCClient-NioEventLoopGroup-3-15 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:23,350 DEBUG [RPCClient-NioEventLoopGroup-3-15 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:23,358 DEBUG [FileChangeWatcher-hbase_test_key11389729334378219996.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key11389729334378219996.bcfks thread started 2025-01-10T07:54:23,358 DEBUG [RPCClient-NioEventLoopGroup-3-15 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:23,358 DEBUG [FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks thread started 2025-01-10T07:54:23,511 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:23,648 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:23,649 DEBUG [FileChangeWatcher-hbase_test_key11389729334378219996.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key11389729334378219996.bcfks thread started 2025-01-10T07:54:23,649 DEBUG [FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks thread started 2025-01-10T07:54:23,649 DEBUG [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0xb075cbf3, L:/127.0.0.1:36011 - R:/127.0.0.1:39132] 2025-01-10T07:54:23,849 INFO [TestNettyTLSIPCFileWatcher-1-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39132, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=TestProtobufRpcProto 2025-01-10T07:54:23,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:23,852 DEBUG [FileChangeWatcher-hbase_test_key11389729334378219996.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:23,852 DEBUG [FileChangeWatcher-hbase_test_key11389729334378219996.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key11389729334378219996.bcfks thread finished 2025-01-10T07:54:23,852 DEBUG [FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:23,852 DEBUG [FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks thread finished 2025-01-10T07:54:24,271 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dde4a70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-10T07:54:24,272 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2025-01-10T07:54:24,272 DEBUG [RPCClient-NioEventLoopGroup-3-16 {}] tls.X509Util(314): Using default JDK SSL provider because netty-tcnative is not available 2025-01-10T07:54:24,573 DEBUG [RPCClient-NioEventLoopGroup-3-16 {}] tls.X509Util(237): Using Java11+ optimized cipher suites for Java version 17, including TLSv1.3 support 2025-01-10T07:54:24,574 DEBUG [FileChangeWatcher-hbase_test_key11389729334378219996.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_key11389729334378219996.bcfks thread started 2025-01-10T07:54:24,575 DEBUG [FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks {}] io.FileChangeWatcher$WatcherThread(202): FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks thread started 2025-01-10T07:54:24,575 DEBUG [RPCClient-NioEventLoopGroup-3-16 {}] ipc.NettyRpcConnection$3(383): SSL handler added with handshake timeout 5000 ms 2025-01-10T07:54:24,576 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(449): SSL handler added for channel: [id: 0x72de4863, L:/127.0.0.1:36011 - R:/127.0.0.1:39134] 2025-01-10T07:54:24,585 WARN [RPCClient-NioEventLoopGroup-3-16 {}] ipc.NettyRpcConnection$2(410): Exception encountered while connecting to the server localhost:36011 javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] ... 31 more 2025-01-10T07:54:24,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-10T07:54:24,586 INFO [Time-limited test {}] ipc.NettyRpcServer(351): Stopping server on localhost/127.0.0.1:36011 2025-01-10T07:54:24,586 DEBUG [FileChangeWatcher-hbase_test_key11389729334378219996.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:24,586 DEBUG [FileChangeWatcher-hbase_test_key11389729334378219996.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:24,586 DEBUG [FileChangeWatcher-hbase_test_key11389729334378219996.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key11389729334378219996.bcfks thread finished 2025-01-10T07:54:24,586 DEBUG [FileChangeWatcher-hbase_test_key11389729334378219996.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_key11389729334378219996.bcfks thread finished 2025-01-10T07:54:24,586 DEBUG [FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:24,586 DEBUG [FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks thread finished 2025-01-10T07:54:24,586 DEBUG [RPCClient-NioEventLoopGroup-3-16 {}] ipc.FailedServers(52): Added failed server with address localhost:36011 to list caused by javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed 2025-01-10T07:54:24,586 DEBUG [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServer(470): Could not get peer certificate on TLS connection from /127.0.0.1:39134, although one is required javax.net.ssl.SSLPeerUnverifiedException: peer not authenticated at sun.security.ssl.SSLSessionImpl.getPeerCertificates(SSLSessionImpl.java:1034) ~[?:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.sslHandshakeCompleteHandler(NettyRpcServer.java:456) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServer.lambda$initSSL$0(NettyRpcServer.java:446) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.handleUnwrapThrowable(SslHandler.java:1381) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1363) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2025-01-10T07:54:24,587 DEBUG [RPCClient-NioEventLoopGroup-3-16 {}] ipc.BufferCallBeforeInitHandler(140): got ssl exception, which should have already been proceeded, log it here to prevent it being passed to netty's TailContext and trigger a confusing WARN message org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:378) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:321) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:316) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:654) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.cert.CertPathValidatorException: signature check failed at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more Caused by: java.security.SignatureException: Signature does not match. at sun.security.x509.X509CertImpl.verify(X509CertImpl.java:450) ~[?:?] at sun.security.provider.certpath.BasicChecker.verifySignature(BasicChecker.java:166) ~[?:?] at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:147) ~[?:?] at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:224) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:144) ~[?:?] at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:83) ~[?:?] at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?] at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?] at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?] at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?] at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?] at org.apache.hadoop.hbase.io.crypto.tls.HBaseTrustManager.checkServerTrusted(HBaseTrustManager.java:112) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkServerCerts(CertificateMessage.java:632) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.onCertificate(CertificateMessage.java:473) ~[?:?] at sun.security.ssl.CertificateMessage$T12CertificateConsumer.consume(CertificateMessage.java:369) ~[?:?] at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?] at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264) ~[?:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1671) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1517) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 17 more 2025-01-10T07:54:24,587 WARN [TestNettyTLSIPCFileWatcher-1-3 {}] ipc.NettyRpcServerPreambleHandler(100): Connection /127.0.0.1:39134; caught unexpected downstream exception. org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:500) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] at sun.security.ssl.Alert.createSSLException(Alert.java:117) ~[?:?] at sun.security.ssl.TransportContext.fatal(TransportContext.java:365) ~[?:?] at sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:293) ~[?:?] at sun.security.ssl.TransportContext.dispatch(TransportContext.java:204) ~[?:?] at sun.security.ssl.SSLTransport.decode(SSLTransport.java:172) ~[?:?] at sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:736) ~[?:?] at sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:691) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:506) ~[?:?] at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:482) ~[?:?] at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:679) ~[?:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:308) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1465) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1358) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1407) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[hbase-shaded-netty-4.1.9.jar:?] ... 15 more 2025-01-10T07:54:24,586 DEBUG [FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks {}] io.FileChangeWatcher$WatcherThread(248): Interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.runLoop(FileChangeWatcher.java:246) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.FileChangeWatcher$WatcherThread.run(FileChangeWatcher.java:214) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2025-01-10T07:54:24,588 DEBUG [FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks {}] io.FileChangeWatcher$WatcherThread(219): FileChangeWatcher-hbase_test_ca1355949995670972225.bcfks thread finished 2025-01-10T07:54:24,597 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestNettyTLSIPCFileWatcher#testReplaceServerKeystore[7: keyType=EC, storeFileType=BCFKS] Thread=35 (was 35), OpenFileDescriptor=327 (was 327), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=795 (was 795), ProcessCount=11 (was 11), AvailableMemoryMB=3350 (was 3386)