2024-11-18 23:54:26,266 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-18 23:54:26,278 main DEBUG Took 0.009966 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-18 23:54:26,278 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-18 23:54:26,278 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-18 23:54:26,279 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-18 23:54:26,287 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:26,296 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-18 23:54:26,311 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:26,312 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:26,313 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:26,314 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:26,314 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:26,315 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:26,316 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:26,316 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:26,317 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:26,317 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:26,318 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:26,318 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:26,319 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:26,320 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:26,320 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:26,321 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:26,321 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:26,322 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:26,322 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:26,323 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:26,323 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:26,324 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:26,324 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:26,325 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:26,325 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:26,326 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-18 23:54:26,328 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:26,329 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-18 23:54:26,331 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-18 23:54:26,332 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-18 23:54:26,333 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-18 23:54:26,334 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-18 23:54:26,344 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-18 23:54:26,347 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-18 23:54:26,349 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-18 23:54:26,350 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-18 23:54:26,350 main DEBUG createAppenders(={Console}) 2024-11-18 23:54:26,351 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e initialized 2024-11-18 23:54:26,351 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-18 23:54:26,351 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e OK. 2024-11-18 23:54:26,352 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-18 23:54:26,353 main DEBUG OutputStream closed 2024-11-18 23:54:26,353 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-18 23:54:26,353 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-18 23:54:26,354 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@74751b3 OK 2024-11-18 23:54:26,438 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-18 23:54:26,439 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-18 23:54:26,441 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-18 23:54:26,442 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-18 23:54:26,443 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-18 23:54:26,443 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-18 23:54:26,444 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-18 23:54:26,444 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-18 23:54:26,445 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-18 23:54:26,445 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-18 23:54:26,446 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-18 23:54:26,446 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-18 23:54:26,446 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-18 23:54:26,447 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-18 23:54:26,447 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-18 23:54:26,447 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-18 23:54:26,448 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-18 23:54:26,448 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-18 23:54:26,450 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18 23:54:26,451 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@528c868) with optional ClassLoader: null 2024-11-18 23:54:26,451 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-18 23:54:26,452 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@528c868] started OK. 2024-11-18T23:54:26,466 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose timeout: 13 mins 2024-11-18 23:54:26,470 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-18 23:54:26,470 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18T23:54:26,718 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941 2024-11-18T23:54:26,742 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=98, ProcessCount=11, AvailableMemoryMB=5235 2024-11-18T23:54:26,847 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-18T23:54:26,971 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:26,987 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/cluster_f5cb780e-0973-b883-9736-9e57669f0008, deleteOnExit=true 2024-11-18T23:54:27,099 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/cluster_f5cb780e-0973-b883-9736-9e57669f0008/zookeeper_0, clientPort=58303, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/cluster_f5cb780e-0973-b883-9736-9e57669f0008/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/cluster_f5cb780e-0973-b883-9736-9e57669f0008/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T23:54:27,113 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58303 2024-11-18T23:54:27,114 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:27,204 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d with version=8 2024-11-18T23:54:27,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/hbase-staging 2024-11-18T23:54:27,305 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-18T23:54:27,586 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9c732c6543ff:0 server-side Connection retries=45 2024-11-18T23:54:27,597 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:27,599 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:27,610 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T23:54:27,610 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:27,610 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T23:54:27,754 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T23:54:27,862 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T23:54:27,901 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-18T23:54:27,912 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-18T23:54:27,916 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T23:54:27,956 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 22252 (auto-detected) 2024-11-18T23:54:27,957 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-18T23:54:27,978 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36819 2024-11-18T23:54:28,001 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36819 connecting to ZooKeeper ensemble=127.0.0.1:58303 2024-11-18T23:54:28,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:368190x0, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T23:54:28,040 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36819-0x1010845e8290000 connected 2024-11-18T23:54:28,075 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:28,076 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:28,086 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:54:28,090 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d, hbase.cluster.distributed=false 2024-11-18T23:54:28,118 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T23:54:28,126 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36819 2024-11-18T23:54:28,127 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36819 2024-11-18T23:54:28,129 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36819 2024-11-18T23:54:28,130 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36819 2024-11-18T23:54:28,130 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36819 2024-11-18T23:54:28,220 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9c732c6543ff:0 server-side Connection retries=45 2024-11-18T23:54:28,222 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:28,222 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:28,222 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T23:54:28,223 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:28,223 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T23:54:28,226 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T23:54:28,228 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T23:54:28,229 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42993 2024-11-18T23:54:28,231 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42993 connecting to ZooKeeper ensemble=127.0.0.1:58303 2024-11-18T23:54:28,232 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:28,233 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:28,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:429930x0, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T23:54:28,237 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42993-0x1010845e8290001 connected 2024-11-18T23:54:28,238 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42993-0x1010845e8290001, quorum=127.0.0.1:58303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:54:28,244 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T23:54:28,258 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/bucket.cache 2024-11-18T23:54:28,266 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T23:54:28,268 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T23:54:28,270 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2024-11-18T23:54:28,272 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T23:54:28,272 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-18T23:54:28,280 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T23:54:28,284 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T23:54:28,284 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/bucket.persistence is complete. 2024-11-18T23:54:28,287 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T23:54:28,290 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42993-0x1010845e8290001, quorum=127.0.0.1:58303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T23:54:28,298 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42993-0x1010845e8290001, quorum=127.0.0.1:58303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T23:54:28,298 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42993 2024-11-18T23:54:28,301 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42993 2024-11-18T23:54:28,302 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42993 2024-11-18T23:54:28,303 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42993 2024-11-18T23:54:28,303 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42993 2024-11-18T23:54:28,321 DEBUG [M:0;9c732c6543ff:36819 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9c732c6543ff:36819 2024-11-18T23:54:28,325 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9c732c6543ff,36819,1731974067385 2024-11-18T23:54:28,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x1010845e8290001, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:54:28,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:54:28,336 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9c732c6543ff,36819,1731974067385 2024-11-18T23:54:28,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:28,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x1010845e8290001, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T23:54:28,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x1010845e8290001, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:28,371 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T23:54:28,372 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9c732c6543ff,36819,1731974067385 from backup master directory 2024-11-18T23:54:28,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9c732c6543ff,36819,1731974067385 2024-11-18T23:54:28,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:54:28,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x1010845e8290001, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:54:28,377 WARN [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T23:54:28,377 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9c732c6543ff,36819,1731974067385 2024-11-18T23:54:28,380 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-18T23:54:28,383 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-18T23:54:28,393 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/hbase.id] with ID: 22384249-662e-41e0-83a5-8f417afeed43 2024-11-18T23:54:28,393 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/.tmp/hbase.id 2024-11-18T23:54:28,400 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/hbase.id] 2024-11-18T23:54:28,447 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:28,452 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T23:54:28,480 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 25ms. 2024-11-18T23:54:28,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:28,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x1010845e8290001, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:28,521 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T23:54:28,526 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T23:54:28,576 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:54:28,582 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:54:28,646 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/data/master/store 2024-11-18T23:54:28,668 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-18T23:54:28,671 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:28,673 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T23:54:28,673 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:28,673 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:28,674 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T23:54:28,675 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:28,675 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:28,675 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731974068672Disabling compacts and flushes for region at 1731974068673 (+1 ms)Disabling writes for close at 1731974068675 (+2 ms)Writing region close event to WAL at 1731974068675Closed at 1731974068675 2024-11-18T23:54:28,676 WARN [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/data/master/store/.initializing 2024-11-18T23:54:28,676 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/WALs/9c732c6543ff,36819,1731974067385 2024-11-18T23:54:28,690 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:54:28,704 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C36819%2C1731974067385, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/WALs/9c732c6543ff,36819,1731974067385, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/oldWALs, maxLogs=10 2024-11-18T23:54:28,743 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/WALs/9c732c6543ff,36819,1731974067385/9c732c6543ff%2C36819%2C1731974067385.1731974068708 2024-11-18T23:54:28,744 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T23:54:28,745 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:54:28,745 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:28,749 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:28,751 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:28,794 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:28,827 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T23:54:28,831 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:28,833 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:28,834 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:28,839 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T23:54:28,840 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:28,840 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:54:28,841 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:28,846 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T23:54:28,846 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:28,847 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:54:28,847 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:28,852 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T23:54:28,852 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:28,853 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:54:28,853 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:28,858 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:28,859 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:28,863 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:28,863 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:28,867 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T23:54:28,870 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:28,879 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:54:28,880 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73298318, jitterRate=0.09223005175590515}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T23:54:28,889 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731974068767Initializing all the Stores at 1731974068768 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974068769 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974068770 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974068770Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974068771 (+1 ms)Cleaning up temporary data from old regions at 1731974068864 (+93 ms)Region opened successfully at 1731974068889 (+25 ms) 2024-11-18T23:54:28,891 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T23:54:28,935 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@590551f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0 2024-11-18T23:54:28,983 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T23:54:29,001 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-18T23:54:29,001 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-18T23:54:29,006 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T23:54:29,007 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T23:54:29,015 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-18T23:54:29,015 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T23:54:29,051 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T23:54:29,064 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T23:54:29,067 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T23:54:29,069 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T23:54:29,071 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T23:54:29,073 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T23:54:29,075 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T23:54:29,079 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T23:54:29,081 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T23:54:29,083 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T23:54:29,085 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T23:54:29,105 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T23:54:29,107 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T23:54:29,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x1010845e8290001, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T23:54:29,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T23:54:29,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x1010845e8290001, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:29,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:29,125 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9c732c6543ff,36819,1731974067385, sessionid=0x1010845e8290000, setting cluster-up flag (Was=false) 2024-11-18T23:54:29,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:29,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x1010845e8290001, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:29,146 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T23:54:29,148 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9c732c6543ff,36819,1731974067385 2024-11-18T23:54:29,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:29,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x1010845e8290001, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:29,158 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T23:54:29,160 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9c732c6543ff,36819,1731974067385 2024-11-18T23:54:29,164 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T23:54:29,209 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.HRegionServer(746): ClusterId : 22384249-662e-41e0-83a5-8f417afeed43 2024-11-18T23:54:29,213 DEBUG [RS:0;9c732c6543ff:42993 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T23:54:29,219 DEBUG [RS:0;9c732c6543ff:42993 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T23:54:29,219 DEBUG [RS:0;9c732c6543ff:42993 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T23:54:29,223 DEBUG [RS:0;9c732c6543ff:42993 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T23:54:29,224 DEBUG [RS:0;9c732c6543ff:42993 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7addb0b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0 2024-11-18T23:54:29,241 DEBUG [RS:0;9c732c6543ff:42993 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9c732c6543ff:42993 2024-11-18T23:54:29,244 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T23:54:29,245 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T23:54:29,245 DEBUG [RS:0;9c732c6543ff:42993 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T23:54:29,247 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T23:54:29,248 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.HRegionServer(2659): reportForDuty to master=9c732c6543ff,36819,1731974067385 with port=42993, startcode=1731974068179 2024-11-18T23:54:29,261 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T23:54:29,262 DEBUG [RS:0;9c732c6543ff:42993 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T23:54:29,269 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T23:54:29,276 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9c732c6543ff,36819,1731974067385 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T23:54:29,283 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5 2024-11-18T23:54:29,283 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5 2024-11-18T23:54:29,283 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5 2024-11-18T23:54:29,284 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5 2024-11-18T23:54:29,284 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9c732c6543ff:0, corePoolSize=10, maxPoolSize=10 2024-11-18T23:54:29,284 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:29,284 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2 2024-11-18T23:54:29,284 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:29,287 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731974099287 2024-11-18T23:54:29,288 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T23:54:29,289 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T23:54:29,289 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T23:54:29,290 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T23:54:29,292 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T23:54:29,293 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T23:54:29,293 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:29,293 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T23:54:29,294 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T23:54:29,294 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T23:54:29,294 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:29,300 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T23:54:29,301 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T23:54:29,302 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T23:54:29,304 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T23:54:29,304 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T23:54:29,306 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974069306,5,FailOnTimeoutGroup] 2024-11-18T23:54:29,309 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974069306,5,FailOnTimeoutGroup] 2024-11-18T23:54:29,309 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:29,310 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T23:54:29,311 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:29,311 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:29,321 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T23:54:29,322 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d 2024-11-18T23:54:29,335 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:29,337 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T23:54:29,345 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T23:54:29,345 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:29,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:29,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T23:54:29,348 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50129, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T23:54:29,350 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T23:54:29,350 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:29,351 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:29,351 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T23:54:29,356 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36819 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9c732c6543ff,42993,1731974068179 2024-11-18T23:54:29,358 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T23:54:29,358 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:29,358 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:29,359 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T23:54:29,359 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36819 {}] master.ServerManager(517): Registering regionserver=9c732c6543ff,42993,1731974068179 2024-11-18T23:54:29,364 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T23:54:29,364 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:29,364 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:29,364 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T23:54:29,365 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740 2024-11-18T23:54:29,365 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740 2024-11-18T23:54:29,366 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T23:54:29,366 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T23:54:29,366 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T23:54:29,367 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T23:54:29,375 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:54:29,376 DEBUG [RS:0;9c732c6543ff:42993 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d 2024-11-18T23:54:29,376 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73985048, jitterRate=0.10246312618255615}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T23:54:29,376 DEBUG [RS:0;9c732c6543ff:42993 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T23:54:29,376 DEBUG [RS:0;9c732c6543ff:42993 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T23:54:29,379 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731974069336Initializing all the Stores at 1731974069336Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974069336Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974069337 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974069337Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974069337Cleaning up temporary data from old regions at 1731974069366 (+29 ms)Region opened successfully at 1731974069379 (+13 ms) 2024-11-18T23:54:29,379 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T23:54:29,379 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T23:54:29,379 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T23:54:29,379 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T23:54:29,379 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T23:54:29,381 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T23:54:29,381 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731974069379Disabling compacts and flushes for region at 1731974069379Disabling writes for close at 1731974069379Writing region close event to WAL at 1731974069380 (+1 ms)Closed at 1731974069381 (+1 ms) 2024-11-18T23:54:29,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T23:54:29,382 DEBUG [RS:0;9c732c6543ff:42993 {}] zookeeper.ZKUtil(111): regionserver:42993-0x1010845e8290001, quorum=127.0.0.1:58303, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9c732c6543ff,42993,1731974068179 2024-11-18T23:54:29,382 WARN [RS:0;9c732c6543ff:42993 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T23:54:29,382 INFO [RS:0;9c732c6543ff:42993 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:54:29,383 DEBUG [RS:0;9c732c6543ff:42993 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/WALs/9c732c6543ff,42993,1731974068179 2024-11-18T23:54:29,384 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T23:54:29,384 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T23:54:29,387 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9c732c6543ff,42993,1731974068179] 2024-11-18T23:54:29,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T23:54:29,401 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T23:54:29,404 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T23:54:29,413 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T23:54:29,428 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T23:54:29,433 INFO [RS:0;9c732c6543ff:42993 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T23:54:29,433 INFO [RS:0;9c732c6543ff:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:29,434 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T23:54:29,439 INFO [RS:0;9c732c6543ff:42993 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T23:54:29,440 INFO [RS:0;9c732c6543ff:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:29,440 DEBUG [RS:0;9c732c6543ff:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:29,441 DEBUG [RS:0;9c732c6543ff:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:29,441 DEBUG [RS:0;9c732c6543ff:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:29,441 DEBUG [RS:0;9c732c6543ff:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:29,441 DEBUG [RS:0;9c732c6543ff:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:29,441 DEBUG [RS:0;9c732c6543ff:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2 2024-11-18T23:54:29,441 DEBUG [RS:0;9c732c6543ff:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:29,442 DEBUG [RS:0;9c732c6543ff:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:29,442 DEBUG [RS:0;9c732c6543ff:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:29,442 DEBUG [RS:0;9c732c6543ff:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:29,442 DEBUG [RS:0;9c732c6543ff:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:29,442 DEBUG [RS:0;9c732c6543ff:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:29,442 DEBUG [RS:0;9c732c6543ff:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3 2024-11-18T23:54:29,442 DEBUG [RS:0;9c732c6543ff:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3 2024-11-18T23:54:29,446 INFO [RS:0;9c732c6543ff:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:29,446 INFO [RS:0;9c732c6543ff:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:29,447 INFO [RS:0;9c732c6543ff:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:29,447 INFO [RS:0;9c732c6543ff:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:29,447 INFO [RS:0;9c732c6543ff:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:29,447 INFO [RS:0;9c732c6543ff:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,42993,1731974068179-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T23:54:29,481 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T23:54:29,484 INFO [RS:0;9c732c6543ff:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,42993,1731974068179-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:29,485 INFO [RS:0;9c732c6543ff:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:29,485 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.Replication(171): 9c732c6543ff,42993,1731974068179 started 2024-11-18T23:54:29,506 INFO [RS:0;9c732c6543ff:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:29,506 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.HRegionServer(1482): Serving as 9c732c6543ff,42993,1731974068179, RpcServer on 9c732c6543ff/172.17.0.2:42993, sessionid=0x1010845e8290001 2024-11-18T23:54:29,507 DEBUG [RS:0;9c732c6543ff:42993 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T23:54:29,508 DEBUG [RS:0;9c732c6543ff:42993 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9c732c6543ff,42993,1731974068179 2024-11-18T23:54:29,508 DEBUG [RS:0;9c732c6543ff:42993 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,42993,1731974068179' 2024-11-18T23:54:29,508 DEBUG [RS:0;9c732c6543ff:42993 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T23:54:29,510 DEBUG [RS:0;9c732c6543ff:42993 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T23:54:29,511 DEBUG [RS:0;9c732c6543ff:42993 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T23:54:29,511 DEBUG [RS:0;9c732c6543ff:42993 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T23:54:29,511 DEBUG [RS:0;9c732c6543ff:42993 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9c732c6543ff,42993,1731974068179 2024-11-18T23:54:29,511 DEBUG [RS:0;9c732c6543ff:42993 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,42993,1731974068179' 2024-11-18T23:54:29,512 DEBUG [RS:0;9c732c6543ff:42993 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T23:54:29,513 DEBUG [RS:0;9c732c6543ff:42993 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T23:54:29,513 DEBUG [RS:0;9c732c6543ff:42993 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T23:54:29,513 INFO [RS:0;9c732c6543ff:42993 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T23:54:29,514 INFO [RS:0;9c732c6543ff:42993 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T23:54:29,555 WARN [9c732c6543ff:36819 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T23:54:29,620 INFO [RS:0;9c732c6543ff:42993 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:54:29,621 INFO [RS:0;9c732c6543ff:42993 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9c732c6543ff%2C42993%2C1731974068179, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/WALs/9c732c6543ff,42993,1731974068179, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/oldWALs, maxLogs=55 2024-11-18T23:54:29,646 INFO [RS:0;9c732c6543ff:42993 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/WALs/9c732c6543ff,42993,1731974068179/9c732c6543ff%2C42993%2C1731974068179.1731974069624 2024-11-18T23:54:29,646 DEBUG [RS:0;9c732c6543ff:42993 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T23:54:29,808 DEBUG [9c732c6543ff:36819 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-18T23:54:29,821 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9c732c6543ff,42993,1731974068179 2024-11-18T23:54:29,827 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,42993,1731974068179, state=OPENING 2024-11-18T23:54:29,832 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T23:54:29,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x1010845e8290001, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:29,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:29,835 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:29,835 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:29,837 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T23:54:29,839 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,42993,1731974068179}] 2024-11-18T23:54:30,012 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T23:54:30,016 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56453, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T23:54:30,024 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T23:54:30,024 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:54:30,025 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T23:54:30,026 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9c732c6543ff%2C42993%2C1731974068179.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/WALs/9c732c6543ff,42993,1731974068179, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/oldWALs, maxLogs=55 2024-11-18T23:54:30,048 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/WALs/9c732c6543ff,42993,1731974068179/9c732c6543ff%2C42993%2C1731974068179.meta.1731974070028.meta 2024-11-18T23:54:30,049 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T23:54:30,049 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:54:30,051 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T23:54:30,054 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T23:54:30,058 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T23:54:30,063 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T23:54:30,063 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:30,063 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T23:54:30,064 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T23:54:30,065 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T23:54:30,066 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T23:54:30,066 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:30,067 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:30,067 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T23:54:30,068 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T23:54:30,068 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:30,068 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:30,068 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T23:54:30,069 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T23:54:30,069 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:30,070 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:30,070 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T23:54:30,071 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T23:54:30,071 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:30,071 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:30,071 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T23:54:30,072 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740 2024-11-18T23:54:30,073 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740 2024-11-18T23:54:30,074 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T23:54:30,074 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T23:54:30,074 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T23:54:30,076 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T23:54:30,076 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70324029, jitterRate=0.0479096919298172}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T23:54:30,077 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T23:54:30,078 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731974070064Writing region info on filesystem at 1731974070064Initializing all the Stores at 1731974070065 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974070065Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974070065Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974070065Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974070065Cleaning up temporary data from old regions at 1731974070074 (+9 ms)Running coprocessor post-open hooks at 1731974070077 (+3 ms)Region opened successfully at 1731974070078 (+1 ms) 2024-11-18T23:54:30,084 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731974070003 2024-11-18T23:54:30,094 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T23:54:30,094 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T23:54:30,095 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9c732c6543ff,42993,1731974068179 2024-11-18T23:54:30,097 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,42993,1731974068179, state=OPEN 2024-11-18T23:54:30,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x1010845e8290001, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T23:54:30,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T23:54:30,102 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:30,102 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:30,102 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9c732c6543ff,42993,1731974068179 2024-11-18T23:54:30,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T23:54:30,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,42993,1731974068179 in 264 msec 2024-11-18T23:54:30,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T23:54:30,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 714 msec 2024-11-18T23:54:30,110 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T23:54:30,110 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T23:54:30,131 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:54:30,133 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,42993,1731974068179, seqNum=-1] 2024-11-18T23:54:30,153 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:54:30,155 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40467, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:54:30,171 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 974 msec 2024-11-18T23:54:30,171 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731974070171, completionTime=-1 2024-11-18T23:54:30,174 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-18T23:54:30,174 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T23:54:30,201 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-18T23:54:30,201 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731974130201 2024-11-18T23:54:30,202 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731974190201 2024-11-18T23:54:30,202 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-18T23:54:30,204 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,36819,1731974067385-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:30,205 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,36819,1731974067385-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:30,205 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,36819,1731974067385-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:30,206 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9c732c6543ff:36819, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:30,207 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:30,207 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:30,213 DEBUG [master/9c732c6543ff:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T23:54:30,233 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.855sec 2024-11-18T23:54:30,235 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T23:54:30,236 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T23:54:30,237 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T23:54:30,237 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T23:54:30,237 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T23:54:30,238 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,36819,1731974067385-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T23:54:30,239 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,36819,1731974067385-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T23:54:30,243 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T23:54:30,244 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T23:54:30,244 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,36819,1731974067385-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:30,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@567b1492, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:54:30,322 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-18T23:54:30,322 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-18T23:54:30,326 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9c732c6543ff,36819,-1 for getting cluster id 2024-11-18T23:54:30,330 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:54:30,354 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22384249-662e-41e0-83a5-8f417afeed43' 2024-11-18T23:54:30,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:54:30,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22384249-662e-41e0-83a5-8f417afeed43" 2024-11-18T23:54:30,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad807d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:54:30,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9c732c6543ff,36819,-1] 2024-11-18T23:54:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:54:30,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:30,369 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:54:30,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a3d5ddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:54:30,373 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:54:30,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,42993,1731974068179, seqNum=-1] 2024-11-18T23:54:30,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:54:30,387 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:54:30,408 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9c732c6543ff,36819,1731974067385 2024-11-18T23:54:30,412 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T23:54:30,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9c732c6543ff,36819,1731974067385 2024-11-18T23:54:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@67050e42 2024-11-18T23:54:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T23:54:30,422 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T23:54:30,428 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36819 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'table1', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T23:54:30,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36819 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-11-18T23:54:30,437 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T23:54:30,439 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36819 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 4 2024-11-18T23:54:30,440 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:30,441 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T23:54:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36819 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:54:30,456 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e7a2516df87b0be8afb85a58cc64c28d, NAME => 'table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d 2024-11-18T23:54:30,462 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:30,463 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing e7a2516df87b0be8afb85a58cc64c28d, disabling compactions & flushes 2024-11-18T23:54:30,463 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d. 2024-11-18T23:54:30,463 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d. 2024-11-18T23:54:30,463 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d. after waiting 0 ms 2024-11-18T23:54:30,463 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d. 2024-11-18T23:54:30,463 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d. 2024-11-18T23:54:30,463 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for e7a2516df87b0be8afb85a58cc64c28d: Waiting for close lock at 1731974070463Disabling compacts and flushes for region at 1731974070463Disabling writes for close at 1731974070463Writing region close event to WAL at 1731974070463Closed at 1731974070463 2024-11-18T23:54:30,465 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T23:54:30,472 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1731974070465"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731974070465"}]},"ts":"1731974070465"} 2024-11-18T23:54:30,477 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T23:54:30,478 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T23:54:30,482 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731974070478"}]},"ts":"1731974070478"} 2024-11-18T23:54:30,486 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-11-18T23:54:30,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=e7a2516df87b0be8afb85a58cc64c28d, ASSIGN}] 2024-11-18T23:54:30,490 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=e7a2516df87b0be8afb85a58cc64c28d, ASSIGN 2024-11-18T23:54:30,491 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=e7a2516df87b0be8afb85a58cc64c28d, ASSIGN; state=OFFLINE, location=9c732c6543ff,42993,1731974068179; forceNewPlan=false, retain=false 2024-11-18T23:54:30,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36819 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:54:30,643 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e7a2516df87b0be8afb85a58cc64c28d, regionState=OPENING, regionLocation=9c732c6543ff,42993,1731974068179 2024-11-18T23:54:30,646 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=e7a2516df87b0be8afb85a58cc64c28d, ASSIGN because future has completed 2024-11-18T23:54:30,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7a2516df87b0be8afb85a58cc64c28d, server=9c732c6543ff,42993,1731974068179}] 2024-11-18T23:54:30,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36819 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:54:30,804 INFO [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d. 2024-11-18T23:54:30,804 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e7a2516df87b0be8afb85a58cc64c28d, NAME => 'table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:54:30,805 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 e7a2516df87b0be8afb85a58cc64c28d 2024-11-18T23:54:30,805 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:30,805 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e7a2516df87b0be8afb85a58cc64c28d 2024-11-18T23:54:30,805 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e7a2516df87b0be8afb85a58cc64c28d 2024-11-18T23:54:30,806 INFO [StoreOpener-e7a2516df87b0be8afb85a58cc64c28d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region e7a2516df87b0be8afb85a58cc64c28d 2024-11-18T23:54:30,811 INFO [StoreOpener-e7a2516df87b0be8afb85a58cc64c28d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7a2516df87b0be8afb85a58cc64c28d columnFamilyName family 2024-11-18T23:54:30,811 DEBUG [StoreOpener-e7a2516df87b0be8afb85a58cc64c28d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:30,812 INFO [StoreOpener-e7a2516df87b0be8afb85a58cc64c28d-1 {}] regionserver.HStore(327): Store=e7a2516df87b0be8afb85a58cc64c28d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:54:30,812 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e7a2516df87b0be8afb85a58cc64c28d 2024-11-18T23:54:30,813 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/default/table1/e7a2516df87b0be8afb85a58cc64c28d 2024-11-18T23:54:30,813 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/default/table1/e7a2516df87b0be8afb85a58cc64c28d 2024-11-18T23:54:30,813 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e7a2516df87b0be8afb85a58cc64c28d 2024-11-18T23:54:30,813 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e7a2516df87b0be8afb85a58cc64c28d 2024-11-18T23:54:30,814 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e7a2516df87b0be8afb85a58cc64c28d 2024-11-18T23:54:30,822 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/default/table1/e7a2516df87b0be8afb85a58cc64c28d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:54:30,822 INFO [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e7a2516df87b0be8afb85a58cc64c28d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71017018, jitterRate=0.05823603272438049}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T23:54:30,822 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7a2516df87b0be8afb85a58cc64c28d 2024-11-18T23:54:30,823 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e7a2516df87b0be8afb85a58cc64c28d: Running coprocessor pre-open hook at 1731974070806Writing region info on filesystem at 1731974070806Initializing all the Stores at 1731974070806Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974070806Cleaning up temporary data from old regions at 1731974070813 (+7 ms)Running coprocessor post-open hooks at 1731974070822 (+9 ms)Region opened successfully at 1731974070823 (+1 ms) 2024-11-18T23:54:30,824 INFO [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d., pid=6, masterSystemTime=1731974070800 2024-11-18T23:54:30,827 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d. 2024-11-18T23:54:30,827 INFO [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d. 2024-11-18T23:54:30,828 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e7a2516df87b0be8afb85a58cc64c28d, regionState=OPEN, openSeqNum=2, regionLocation=9c732c6543ff,42993,1731974068179 2024-11-18T23:54:30,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7a2516df87b0be8afb85a58cc64c28d, server=9c732c6543ff,42993,1731974068179 because future has completed 2024-11-18T23:54:30,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-18T23:54:30,835 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e7a2516df87b0be8afb85a58cc64c28d, server=9c732c6543ff,42993,1731974068179 in 185 msec 2024-11-18T23:54:30,836 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-18T23:54:30,836 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=e7a2516df87b0be8afb85a58cc64c28d, ASSIGN in 346 msec 2024-11-18T23:54:30,837 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T23:54:30,838 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731974070837"}]},"ts":"1731974070837"} 2024-11-18T23:54:30,840 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-11-18T23:54:30,841 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T23:54:30,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 411 msec 2024-11-18T23:54:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36819 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:54:31,080 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table table1 get assigned. Timeout = 60000ms 2024-11-18T23:54:31,080 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-11-18T23:54:31,082 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T23:54:31,088 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table table1 assigned to meta. Checking AM states. 2024-11-18T23:54:31,088 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T23:54:31,089 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table table1 assigned. 2024-11-18T23:54:31,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row1', locateType=CURRENT is [region=table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d., hostname=9c732c6543ff,42993,1731974068179, seqNum=2] 2024-11-18T23:54:31,108 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e7a2516df87b0be8afb85a58cc64c28d 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-18T23:54:31,134 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0beec9d8538640beba9fe9b3fb996a39 2024-11-18T23:54:31,172 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/default/table1/e7a2516df87b0be8afb85a58cc64c28d/.tmp/family/0beec9d8538640beba9fe9b3fb996a39 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-18T23:54:31,177 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/default/table1/e7a2516df87b0be8afb85a58cc64c28d/.tmp/family/0beec9d8538640beba9fe9b3fb996a39 2024-11-18T23:54:31,213 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/default/table1/e7a2516df87b0be8afb85a58cc64c28d/.tmp/family/0beec9d8538640beba9fe9b3fb996a39 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/default/table1/e7a2516df87b0be8afb85a58cc64c28d/family/0beec9d8538640beba9fe9b3fb996a39 2024-11-18T23:54:31,215 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/default/table1/e7a2516df87b0be8afb85a58cc64c28d/family/0beec9d8538640beba9fe9b3fb996a39, delay=919 ms 2024-11-18T23:54:31,216 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/default/table1/e7a2516df87b0be8afb85a58cc64c28d/family/0beec9d8538640beba9fe9b3fb996a39, entries=2, sequenceid=6, filesize=4.9 K 2024-11-18T23:54:31,223 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for e7a2516df87b0be8afb85a58cc64c28d in 113ms, sequenceid=6, compaction requested=false 2024-11-18T23:54:31,224 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-11-18T23:54:31,226 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e7a2516df87b0be8afb85a58cc64c28d: 2024-11-18T23:54:32,136 DEBUG [hfile-prefetch-1731974071215 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0beec9d8538640beba9fe9b3fb996a39_0. 2024-11-18T23:54:32,141 DEBUG [hfile-prefetch-1731974071215 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0beec9d8538640beba9fe9b3fb996a39_117. 2024-11-18T23:54:32,142 DEBUG [hfile-prefetch-1731974071215 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/default/table1/e7a2516df87b0be8afb85a58cc64c28d/family/0beec9d8538640beba9fe9b3fb996a39, with total blocks 2, and data blocks 1 2024-11-18T23:54:32,142 DEBUG [hfile-prefetch-1731974071215 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-18T23:54:32,142 DEBUG [hfile-prefetch-1731974071215 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/default/table1/e7a2516df87b0be8afb85a58cc64c28d/family/0beec9d8538640beba9fe9b3fb996a39, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-18T23:54:32,142 DEBUG [hfile-prefetch-1731974071215 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/default/table1/e7a2516df87b0be8afb85a58cc64c28d/family/0beec9d8538640beba9fe9b3fb996a39. Will sleep 100ms and try the verification again. 2024-11-18T23:54:32,243 DEBUG [hfile-prefetch-1731974071215 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/default/table1/e7a2516df87b0be8afb85a58cc64c28d/family/0beec9d8538640beba9fe9b3fb996a39, with total blocks 2, and data blocks 1 2024-11-18T23:54:32,243 DEBUG [hfile-prefetch-1731974071215 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-18T23:54:32,243 DEBUG [hfile-prefetch-1731974071215 {}] bucket.BucketCache(2293): found block for file 0beec9d8538640beba9fe9b3fb996a39 in the backing map. Acquiring read lock for offset 0 2024-11-18T23:54:32,243 DEBUG [hfile-prefetch-1731974071215 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/default/table1/e7a2516df87b0be8afb85a58cc64c28d/family/0beec9d8538640beba9fe9b3fb996a39 has now been fully cached. 2024-11-18T23:54:32,244 DEBUG [hfile-prefetch-1731974071215 {}] hfile.PrefetchExecutor(121): Prefetch completed for 0beec9d8538640beba9fe9b3fb996a39 2024-11-18T23:54:32,271 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-18T23:54:32,271 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-18T23:54:32,271 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T23:54:32,271 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-18T23:54:32,292 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-18T23:54:32,292 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731974072271 2024-11-18T23:54:32,292 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-18T23:54:33,229 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T23:54:33,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T23:54:33,234 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;9c732c6543ff:42993,5,FailOnTimeoutGroup] 2024-11-18T23:54:33,235 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9c732c6543ff,42993,1731974068179' ***** 2024-11-18T23:54:33,235 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-18T23:54:33,235 INFO [Time-limited test {}] hfile.TestPrefetchRSClose(129): Stopped Region Server 0. 2024-11-18T23:54:33,235 INFO [RS:0;9c732c6543ff:42993 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T23:54:33,235 INFO [RS:0;9c732c6543ff:42993 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/bucket.persistence 2024-11-18T23:54:33,235 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T23:54:33,235 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T23:54:33,235 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T23:54:33,236 INFO [RS:0;9c732c6543ff:42993 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T23:54:33,236 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T23:54:33,246 DEBUG [RS:0;9c732c6543ff:42993 {}] bucket.BucketCache(1487): Thread RS:0;9c732c6543ff:42993 started persisting bucket cache to file 2024-11-18T23:54:33,246 DEBUG [RS:0;9c732c6543ff:42993 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T23:54:33,246 DEBUG [RS:0;9c732c6543ff:42993 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-18T23:54:33,250 DEBUG [RS:0;9c732c6543ff:42993 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-18T23:54:33,250 DEBUG [RS:0;9c732c6543ff:42993 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731974073246 2024-11-18T23:54:33,250 DEBUG [RS:0;9c732c6543ff:42993 {}] bucket.BucketCache(1509): Thread RS:0;9c732c6543ff:42993 finished persisting bucket cache to file, renaming 2024-11-18T23:54:33,250 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T23:54:33,250 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T23:54:33,250 INFO [RS:0;9c732c6543ff:42993 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T23:54:33,250 INFO [RS:0;9c732c6543ff:42993 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T23:54:33,251 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.HRegionServer(3091): Received CLOSE for e7a2516df87b0be8afb85a58cc64c28d 2024-11-18T23:54:33,251 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.HRegionServer(959): stopping server 9c732c6543ff,42993,1731974068179 2024-11-18T23:54:33,251 INFO [RS:0;9c732c6543ff:42993 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T23:54:33,252 INFO [RS:0;9c732c6543ff:42993 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9c732c6543ff:42993. 2024-11-18T23:54:33,252 INFO [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T23:54:33,252 DEBUG [RS:0;9c732c6543ff:42993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:54:33,252 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e7a2516df87b0be8afb85a58cc64c28d, disabling compactions & flushes 2024-11-18T23:54:33,252 INFO [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d. 2024-11-18T23:54:33,252 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d. 2024-11-18T23:54:33,252 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d. after waiting 0 ms 2024-11-18T23:54:33,252 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d. 2024-11-18T23:54:33,255 INFO [regionserver/9c732c6543ff:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T23:54:33,257 DEBUG [RS:0;9c732c6543ff:42993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:33,257 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T23:54:33,257 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T23:54:33,257 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T23:54:33,258 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T23:54:33,258 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-18T23:54:33,258 DEBUG [RS:0;9c732c6543ff:42993 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, e7a2516df87b0be8afb85a58cc64c28d=table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d.} 2024-11-18T23:54:33,258 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T23:54:33,258 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T23:54:33,258 DEBUG [RS:0;9c732c6543ff:42993 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, e7a2516df87b0be8afb85a58cc64c28d 2024-11-18T23:54:33,258 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T23:54:33,259 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T23:54:33,259 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T23:54:33,259 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T23:54:33,259 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.20 KB heapSize=3.23 KB 2024-11-18T23:54:33,259 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/default/table1/e7a2516df87b0be8afb85a58cc64c28d/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-18T23:54:33,260 INFO [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d. 2024-11-18T23:54:33,261 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e7a2516df87b0be8afb85a58cc64c28d: Waiting for close lock at 1731974073251Running coprocessor pre-close hooks at 1731974073252 (+1 ms)Disabling compacts and flushes for region at 1731974073252Disabling writes for close at 1731974073252Writing region close event to WAL at 1731974073254 (+2 ms)Running coprocessor post-close hooks at 1731974073259 (+5 ms)Closed at 1731974073260 (+1 ms) 2024-11-18T23:54:33,261 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d. 2024-11-18T23:54:33,270 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4973eb4795294b6283bf75c0648f8fff 2024-11-18T23:54:33,286 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/.tmp/info/4973eb4795294b6283bf75c0648f8fff is 133, key is table1,,1731974070424.e7a2516df87b0be8afb85a58cc64c28d./info:regioninfo/1731974070827/Put/seqid=0 2024-11-18T23:54:33,287 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/.tmp/info/4973eb4795294b6283bf75c0648f8fff 2024-11-18T23:54:33,296 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d0c3eb4ef9ff47f18217310f58a4e6e4 2024-11-18T23:54:33,310 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/.tmp/ns/d0c3eb4ef9ff47f18217310f58a4e6e4 is 43, key is default/ns:d/1731974070159/Put/seqid=0 2024-11-18T23:54:33,310 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/.tmp/ns/d0c3eb4ef9ff47f18217310f58a4e6e4 2024-11-18T23:54:33,319 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a1d83fd3600f4dcdbc7ef5199e02a81c 2024-11-18T23:54:33,332 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/.tmp/table/a1d83fd3600f4dcdbc7ef5199e02a81c is 42, key is table1/table:state/1731974070837/Put/seqid=0 2024-11-18T23:54:33,333 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=76 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/.tmp/table/a1d83fd3600f4dcdbc7ef5199e02a81c 2024-11-18T23:54:33,335 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/.tmp/info/4973eb4795294b6283bf75c0648f8fff as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/info/4973eb4795294b6283bf75c0648f8fff 2024-11-18T23:54:33,337 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/info/4973eb4795294b6283bf75c0648f8fff, delay=996 ms 2024-11-18T23:54:33,338 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/info/4973eb4795294b6283bf75c0648f8fff, entries=10, sequenceid=11, filesize=6.3 K 2024-11-18T23:54:33,339 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/.tmp/ns/d0c3eb4ef9ff47f18217310f58a4e6e4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/ns/d0c3eb4ef9ff47f18217310f58a4e6e4 2024-11-18T23:54:33,340 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/ns/d0c3eb4ef9ff47f18217310f58a4e6e4, delay=908 ms 2024-11-18T23:54:33,341 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/ns/d0c3eb4ef9ff47f18217310f58a4e6e4, entries=2, sequenceid=11, filesize=5.0 K 2024-11-18T23:54:33,342 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/.tmp/table/a1d83fd3600f4dcdbc7ef5199e02a81c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/table/a1d83fd3600f4dcdbc7ef5199e02a81c 2024-11-18T23:54:33,343 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/table/a1d83fd3600f4dcdbc7ef5199e02a81c, delay=941 ms 2024-11-18T23:54:33,344 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/table/a1d83fd3600f4dcdbc7ef5199e02a81c, entries=2, sequenceid=11, filesize=5.1 K 2024-11-18T23:54:33,345 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 86ms, sequenceid=11, compaction requested=false 2024-11-18T23:54:33,345 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-18T23:54:33,347 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/info/4973eb4795294b6283bf75c0648f8fff 2024-11-18T23:54:33,347 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/ns/d0c3eb4ef9ff47f18217310f58a4e6e4 2024-11-18T23:54:33,348 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/table/a1d83fd3600f4dcdbc7ef5199e02a81c 2024-11-18T23:54:33,353 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-18T23:54:33,354 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T23:54:33,354 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T23:54:33,354 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731974073258Running coprocessor pre-close hooks at 1731974073258Disabling compacts and flushes for region at 1731974073258Disabling writes for close at 1731974073259 (+1 ms)Obtaining lock to block concurrent updates at 1731974073259Preparing flush snapshotting stores in 1588230740 at 1731974073259Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1227, getHeapSize=3248, getOffHeapSize=0, getCellsCount=14 at 1731974073260 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731974073261 (+1 ms)Flushing 1588230740/info: creating writer at 1731974073261Flushing 1588230740/info: appending metadata at 1731974073285 (+24 ms)Flushing 1588230740/info: closing flushed file at 1731974073285Flushing 1588230740/ns: creating writer at 1731974073290 (+5 ms)Flushing 1588230740/ns: appending metadata at 1731974073309 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731974073309Flushing 1588230740/table: creating writer at 1731974073313 (+4 ms)Flushing 1588230740/table: appending metadata at 1731974073331 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731974073331Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bcf153f: reopening flushed file at 1731974073335 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5289bd27: reopening flushed file at 1731974073338 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a92ccb6: reopening flushed file at 1731974073342 (+4 ms)Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 86ms, sequenceid=11, compaction requested=false at 1731974073345 (+3 ms)Writing region close event to WAL at 1731974073348 (+3 ms)Running coprocessor post-close hooks at 1731974073353 (+5 ms)Closed at 1731974073354 (+1 ms) 2024-11-18T23:54:33,354 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T23:54:33,449 INFO [regionserver/9c732c6543ff:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-18T23:54:33,449 INFO [regionserver/9c732c6543ff:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-18T23:54:33,459 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.HRegionServer(976): stopping server 9c732c6543ff,42993,1731974068179; all regions closed. 2024-11-18T23:54:33,464 DEBUG [RS:0;9c732c6543ff:42993 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/oldWALs 2024-11-18T23:54:33,464 INFO [RS:0;9c732c6543ff:42993 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9c732c6543ff%2C42993%2C1731974068179.meta:.meta(num 1731974070028) 2024-11-18T23:54:33,466 DEBUG [RS:0;9c732c6543ff:42993 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/oldWALs 2024-11-18T23:54:33,466 INFO [RS:0;9c732c6543ff:42993 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9c732c6543ff%2C42993%2C1731974068179:(num 1731974069624) 2024-11-18T23:54:33,466 DEBUG [RS:0;9c732c6543ff:42993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:33,466 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T23:54:33,466 INFO [RS:0;9c732c6543ff:42993 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T23:54:33,466 INFO [RS:0;9c732c6543ff:42993 {}] hbase.ChoreService(370): Chore service for: regionserver/9c732c6543ff:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T23:54:33,466 INFO [RS:0;9c732c6543ff:42993 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T23:54:33,466 INFO [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T23:54:33,467 INFO [RS:0;9c732c6543ff:42993 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42993 2024-11-18T23:54:33,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x1010845e8290001, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9c732c6543ff,42993,1731974068179 2024-11-18T23:54:33,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T23:54:33,473 INFO [RS:0;9c732c6543ff:42993 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T23:54:33,476 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9c732c6543ff,42993,1731974068179] 2024-11-18T23:54:33,479 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9c732c6543ff,42993,1731974068179 already deleted, retry=false 2024-11-18T23:54:33,479 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 9c732c6543ff,42993,1731974068179 on 9c732c6543ff,36819,1731974067385 2024-11-18T23:54:33,484 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 9c732c6543ff,42993,1731974068179, splitWal=true, meta=true 2024-11-18T23:54:33,486 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 9c732c6543ff,42993,1731974068179 (carryingMeta=true) 9c732c6543ff,42993,1731974068179/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@3cfe1f83[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-18T23:54:33,486 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 9c732c6543ff,42993,1731974068179, splitWal=true, meta=true 2024-11-18T23:54:33,487 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 9c732c6543ff,42993,1731974068179, splitWal=true, meta=true, isMeta: true 2024-11-18T23:54:33,488 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/WALs/9c732c6543ff,42993,1731974068179-splitting 2024-11-18T23:54:33,488 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/WALs/9c732c6543ff,42993,1731974068179-splitting dir is empty, no logs to split. 2024-11-18T23:54:33,489 INFO [PEWorker-2 {}] master.SplitWALManager(105): 9c732c6543ff,42993,1731974068179 WAL count=0, meta=true 2024-11-18T23:54:33,491 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/WALs/9c732c6543ff,42993,1731974068179-splitting dir is empty, no logs to split. 2024-11-18T23:54:33,491 INFO [PEWorker-2 {}] master.SplitWALManager(105): 9c732c6543ff,42993,1731974068179 WAL count=0, meta=true 2024-11-18T23:54:33,491 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 9c732c6543ff,42993,1731974068179 WAL splitting is done? wals=0, meta=true 2024-11-18T23:54:33,493 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T23:54:33,494 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T23:54:33,494 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T23:54:33,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x1010845e8290001, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:54:33,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x1010845e8290001, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:54:33,577 INFO [RS:0;9c732c6543ff:42993 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T23:54:33,577 INFO [RS:0;9c732c6543ff:42993 {}] regionserver.HRegionServer(1031): Exiting; stopping=9c732c6543ff,42993,1731974068179; zookeeper connection closed. 2024-11-18T23:54:33,578 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7fdf5e75 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7fdf5e75 2024-11-18T23:54:33,645 WARN [9c732c6543ff:36819 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T23:54:34,235 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T23:54:34,236 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T23:54:34,236 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose.tearDown(TestPrefetchRSClose.java:136) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:54:34,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:34,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:34,236 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:54:34,237 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T23:54:34,237 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=975984163, stopped=false 2024-11-18T23:54:34,237 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9c732c6543ff,36819,1731974067385 2024-11-18T23:54:34,237 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master '9c732c6543ff,36819,1731974067385' ***** 2024-11-18T23:54:34,237 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: OnlineServer=0 right after cluster shutdown set 2024-11-18T23:54:34,238 INFO [M:0;9c732c6543ff:36819 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T23:54:34,238 INFO [M:0;9c732c6543ff:36819 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;9c732c6543ff:36819. 2024-11-18T23:54:34,238 DEBUG [M:0;9c732c6543ff:36819 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:54:34,238 DEBUG [M:0;9c732c6543ff:36819 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:34,238 INFO [M:0;9c732c6543ff:36819 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T23:54:34,238 DEBUG [M:0;9c732c6543ff:36819 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T23:54:34,238 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T23:54:34,238 DEBUG [M:0;9c732c6543ff:36819 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T23:54:34,238 DEBUG [master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974069306 {}] cleaner.HFileCleaner(306): Exit Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974069306,5,FailOnTimeoutGroup] 2024-11-18T23:54:34,238 DEBUG [master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974069306 {}] cleaner.HFileCleaner(306): Exit Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974069306,5,FailOnTimeoutGroup] 2024-11-18T23:54:34,239 INFO [M:0;9c732c6543ff:36819 {}] hbase.ChoreService(370): Chore service for: master/9c732c6543ff:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T23:54:34,239 INFO [M:0;9c732c6543ff:36819 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T23:54:34,239 DEBUG [M:0;9c732c6543ff:36819 {}] master.HMaster(1795): Stopping service threads 2024-11-18T23:54:34,239 INFO [M:0;9c732c6543ff:36819 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T23:54:34,239 INFO [M:0;9c732c6543ff:36819 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T23:54:34,240 INFO [M:0;9c732c6543ff:36819 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T23:54:34,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T23:54:34,240 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T23:54:34,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:34,241 DEBUG [M:0;9c732c6543ff:36819 {}] zookeeper.ZKUtil(347): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-18T23:54:34,241 WARN [M:0;9c732c6543ff:36819 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-18T23:54:34,242 INFO [M:0;9c732c6543ff:36819 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/.lastflushedseqids 2024-11-18T23:54:34,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T23:54:34,243 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-18T23:54:34,243 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:54:34,254 INFO [M:0;9c732c6543ff:36819 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T23:54:34,395 DEBUG [9c732c6543ff:36819 {}] assignment.AssignmentManager(2455): Stopped! Dropping assign of 1 queued regions. 2024-11-18T23:54:34,395 INFO [M:0;9c732c6543ff:36819 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T23:54:34,396 DEBUG [M:0;9c732c6543ff:36819 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T23:54:34,396 INFO [M:0;9c732c6543ff:36819 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:34,396 DEBUG [M:0;9c732c6543ff:36819 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:34,396 DEBUG [M:0;9c732c6543ff:36819 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T23:54:34,396 DEBUG [M:0;9c732c6543ff:36819 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:34,396 INFO [M:0;9c732c6543ff:36819 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=24.18 KB heapSize=30.80 KB 2024-11-18T23:54:34,407 DEBUG [M:0;9c732c6543ff:36819 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a4399d10a91d4ad28bf508840a01d917 2024-11-18T23:54:34,421 DEBUG [M:0;9c732c6543ff:36819 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a4399d10a91d4ad28bf508840a01d917 is 82, key is hbase:meta,,1/info:regioninfo/1731974070095/Put/seqid=0 2024-11-18T23:54:34,422 INFO [M:0;9c732c6543ff:36819 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a4399d10a91d4ad28bf508840a01d917 2024-11-18T23:54:34,431 DEBUG [M:0;9c732c6543ff:36819 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6ccc3b42e1d246c69a769c2678ed22ee 2024-11-18T23:54:34,447 DEBUG [M:0;9c732c6543ff:36819 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6ccc3b42e1d246c69a769c2678ed22ee is 707, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731974070843/Put/seqid=0 2024-11-18T23:54:34,448 INFO [M:0;9c732c6543ff:36819 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.57 KB at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6ccc3b42e1d246c69a769c2678ed22ee 2024-11-18T23:54:34,457 DEBUG [M:0;9c732c6543ff:36819 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on be04c32e79f944e8a68bdd303a9c9f26 2024-11-18T23:54:34,472 DEBUG [M:0;9c732c6543ff:36819 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/be04c32e79f944e8a68bdd303a9c9f26 is 58, key is 9c732c6543ff,42993,1731974068179/rs:/1731974073487/DeleteFamily/seqid=0 2024-11-18T23:54:34,473 INFO [M:0;9c732c6543ff:36819 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=119 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/be04c32e79f944e8a68bdd303a9c9f26 2024-11-18T23:54:34,475 INFO [M:0;9c732c6543ff:36819 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for be04c32e79f944e8a68bdd303a9c9f26 2024-11-18T23:54:34,475 DEBUG [M:0;9c732c6543ff:36819 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a4399d10a91d4ad28bf508840a01d917 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a4399d10a91d4ad28bf508840a01d917 2024-11-18T23:54:34,477 INFO [M:0;9c732c6543ff:36819 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a4399d10a91d4ad28bf508840a01d917, entries=8, sequenceid=64, filesize=5.5 K 2024-11-18T23:54:34,478 DEBUG [M:0;9c732c6543ff:36819 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6ccc3b42e1d246c69a769c2678ed22ee as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6ccc3b42e1d246c69a769c2678ed22ee 2024-11-18T23:54:34,480 INFO [M:0;9c732c6543ff:36819 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6ccc3b42e1d246c69a769c2678ed22ee, entries=8, sequenceid=64, filesize=6.5 K 2024-11-18T23:54:34,480 DEBUG [M:0;9c732c6543ff:36819 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/be04c32e79f944e8a68bdd303a9c9f26 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/be04c32e79f944e8a68bdd303a9c9f26 2024-11-18T23:54:34,482 INFO [M:0;9c732c6543ff:36819 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for be04c32e79f944e8a68bdd303a9c9f26 2024-11-18T23:54:34,482 INFO [M:0;9c732c6543ff:36819 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f7ef6786-05ac-e732-9425-6e464edc7941/47436ea8-041d-1f60-a46a-0ef097efc32d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/be04c32e79f944e8a68bdd303a9c9f26, entries=1, sequenceid=64, filesize=5.2 K 2024-11-18T23:54:34,483 INFO [M:0;9c732c6543ff:36819 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.18 KB/24759, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 86ms, sequenceid=64, compaction requested=false 2024-11-18T23:54:34,485 INFO [M:0;9c732c6543ff:36819 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:34,485 DEBUG [M:0;9c732c6543ff:36819 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731974074396Disabling compacts and flushes for region at 1731974074396Disabling writes for close at 1731974074396Obtaining lock to block concurrent updates at 1731974074396Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731974074396Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=24759, getHeapSize=31480, getOffHeapSize=0, getCellsCount=76 at 1731974074397 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731974074397Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731974074397Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731974074421 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731974074421Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731974074424 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731974074446 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731974074447 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731974074451 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731974074471 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731974074471Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f9f128a: reopening flushed file at 1731974074475 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4efd78b: reopening flushed file at 1731974074478 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b0c70e5: reopening flushed file at 1731974074480 (+2 ms)Finished flush of dataSize ~24.18 KB/24759, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 86ms, sequenceid=64, compaction requested=false at 1731974074483 (+3 ms)Writing region close event to WAL at 1731974074485 (+2 ms)Closed at 1731974074485 2024-11-18T23:54:34,487 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T23:54:34,487 INFO [M:0;9c732c6543ff:36819 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T23:54:34,487 INFO [M:0;9c732c6543ff:36819 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36819 2024-11-18T23:54:34,487 INFO [M:0;9c732c6543ff:36819 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T23:54:34,601 INFO [M:0;9c732c6543ff:36819 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T23:54:34,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:54:34,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36819-0x1010845e8290000, quorum=127.0.0.1:58303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:54:34,606 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T23:54:34,631 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-18T23:54:34,631 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:34,640 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=44 (was 12) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731974071215 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731974073343 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731974073337 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/9c732c6543ff:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731974073340 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/9c732c6543ff:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/9c732c6543ff:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=337 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=98 (was 98), ProcessCount=11 (was 11), AvailableMemoryMB=4915 (was 5235)