2024-12-06 14:43:45,924 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-06 14:43:45,943 main DEBUG Took 0.017515 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-06 14:43:45,943 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-06 14:43:45,944 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-06 14:43:45,944 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-06 14:43:45,945 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:43:45,952 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-06 14:43:45,967 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:43:45,968 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:43:45,969 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:43:45,969 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:43:45,970 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:43:45,970 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:43:45,971 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:43:45,971 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:43:45,972 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:43:45,972 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:43:45,972 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:43:45,973 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:43:45,973 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:43:45,973 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:43:45,974 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:43:45,974 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:43:45,974 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:43:45,975 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:43:45,975 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:43:45,975 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:43:45,976 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:43:45,976 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:43:45,976 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:43:45,976 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 14:43:45,977 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:43:45,977 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-06 14:43:45,978 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 14:43:45,980 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-06 14:43:45,981 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-06 14:43:45,982 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-06 14:43:45,983 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-06 14:43:45,983 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-06 14:43:45,991 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-06 14:43:45,994 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-06 14:43:45,995 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-06 14:43:45,996 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-06 14:43:45,996 main DEBUG createAppenders(={Console}) 2024-12-06 14:43:45,997 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-06 14:43:45,997 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-06 14:43:45,997 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-06 14:43:45,998 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-06 14:43:45,998 main DEBUG OutputStream closed 2024-12-06 14:43:45,998 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-06 14:43:45,999 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-06 14:43:45,999 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-06 14:43:46,067 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-06 14:43:46,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-06 14:43:46,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-06 14:43:46,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-06 14:43:46,072 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-06 14:43:46,072 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-06 14:43:46,072 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-06 14:43:46,073 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-06 14:43:46,073 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-06 14:43:46,073 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-06 14:43:46,074 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-06 14:43:46,074 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-06 14:43:46,074 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-06 14:43:46,074 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-06 14:43:46,075 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-06 14:43:46,075 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-06 14:43:46,075 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-06 14:43:46,076 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-06 14:43:46,078 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-06 14:43:46,079 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-06 14:43:46,079 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-06 14:43:46,080 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-06T14:43:46,092 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-06 14:43:46,095 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-06 14:43:46,095 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-06T14:43:46,322 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb 2024-12-06T14:43:46,342 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=577, ProcessCount=11, AvailableMemoryMB=6849 2024-12-06T14:43:46,451 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-06T14:43:46,548 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:46,564 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/cluster_62ac8d31-5aaa-ae53-1df2-ed494d08ff70, deleteOnExit=true 2024-12-06T14:43:46,667 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/cluster_62ac8d31-5aaa-ae53-1df2-ed494d08ff70/zookeeper_0, clientPort=52310, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/cluster_62ac8d31-5aaa-ae53-1df2-ed494d08ff70/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/cluster_62ac8d31-5aaa-ae53-1df2-ed494d08ff70/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-06T14:43:46,677 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52310 2024-12-06T14:43:46,678 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:46,757 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f with version=8 2024-12-06T14:43:46,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/hbase-staging 2024-12-06T14:43:46,844 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-06T14:43:47,086 INFO [Time-limited test {}] client.ConnectionUtils(128): master/bb7ea3e46d9c:0 server-side Connection retries=45 2024-12-06T14:43:47,094 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:47,095 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:47,102 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T14:43:47,103 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:47,103 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T14:43:47,221 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T14:43:47,293 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T14:43:47,330 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-06T14:43:47,339 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-06T14:43:47,343 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T14:43:47,379 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 14435 (auto-detected) 2024-12-06T14:43:47,380 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-06T14:43:47,402 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44497 2024-12-06T14:43:47,431 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44497 connecting to ZooKeeper ensemble=127.0.0.1:52310 2024-12-06T14:43:47,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:444970x0, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T14:43:47,547 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44497-0x101ab193f050000 connected 2024-12-06T14:43:47,654 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:47,656 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:47,664 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T14:43:47,668 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f, hbase.cluster.distributed=false 2024-12-06T14:43:47,689 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T14:43:47,695 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44497 2024-12-06T14:43:47,695 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44497 2024-12-06T14:43:47,696 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44497 2024-12-06T14:43:47,696 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44497 2024-12-06T14:43:47,697 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44497 2024-12-06T14:43:47,766 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bb7ea3e46d9c:0 server-side Connection retries=45 2024-12-06T14:43:47,768 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:47,768 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:47,769 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T14:43:47,769 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:47,769 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T14:43:47,772 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T14:43:47,774 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T14:43:47,775 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45909 2024-12-06T14:43:47,776 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45909 connecting to ZooKeeper ensemble=127.0.0.1:52310 2024-12-06T14:43:47,777 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:47,777 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:47,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:459090x0, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T14:43:47,791 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45909-0x101ab193f050001 connected 2024-12-06T14:43:47,791 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T14:43:47,798 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T14:43:47,809 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/bucket.cache 2024-12-06T14:43:47,814 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-06T14:43:47,816 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-06T14:43:47,817 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-06T14:43:47,819 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-06T14:43:47,819 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-06T14:43:47,826 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-06T14:43:47,829 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-06T14:43:47,830 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/bucket.persistence is complete. 2024-12-06T14:43:47,833 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T14:43:47,836 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T14:43:47,843 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T14:43:47,843 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45909 2024-12-06T14:43:47,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45909 2024-12-06T14:43:47,846 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45909 2024-12-06T14:43:47,848 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45909 2024-12-06T14:43:47,848 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45909 2024-12-06T14:43:47,864 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bb7ea3e46d9c:0 server-side Connection retries=45 2024-12-06T14:43:47,864 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:47,865 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:47,865 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T14:43:47,865 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:47,865 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T14:43:47,865 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T14:43:47,866 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T14:43:47,866 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42857 2024-12-06T14:43:47,868 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42857 connecting to ZooKeeper ensemble=127.0.0.1:52310 2024-12-06T14:43:47,869 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:47,869 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:47,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:428570x0, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T14:43:47,880 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42857-0x101ab193f050002 connected 2024-12-06T14:43:47,880 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:428570x0, quorum=127.0.0.1:52310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T14:43:47,881 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T14:43:47,883 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/bucket.cache 2024-12-06T14:43:47,883 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-06T14:43:47,883 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-06T14:43:47,884 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-06T14:43:47,884 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-06T14:43:47,884 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-06T14:43:47,884 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-06T14:43:47,885 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-06T14:43:47,885 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/bucket.persistence is complete. 2024-12-06T14:43:47,885 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T14:43:47,887 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42857-0x101ab193f050002, quorum=127.0.0.1:52310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T14:43:47,889 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42857-0x101ab193f050002, quorum=127.0.0.1:52310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T14:43:47,891 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42857 2024-12-06T14:43:47,892 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42857 2024-12-06T14:43:47,894 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42857 2024-12-06T14:43:47,895 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42857 2024-12-06T14:43:47,895 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42857 2024-12-06T14:43:47,909 DEBUG [M:0;bb7ea3e46d9c:44497 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;bb7ea3e46d9c:44497 2024-12-06T14:43:47,910 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/bb7ea3e46d9c,44497,1733496226926 2024-12-06T14:43:47,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T14:43:47,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101ab193f050002, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T14:43:47,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T14:43:47,929 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/bb7ea3e46d9c,44497,1733496226926 2024-12-06T14:43:47,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T14:43:47,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101ab193f050002, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T14:43:47,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:47,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101ab193f050002, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:47,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:47,966 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T14:43:47,967 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/bb7ea3e46d9c,44497,1733496226926 from backup master directory 2024-12-06T14:43:47,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/bb7ea3e46d9c,44497,1733496226926 2024-12-06T14:43:47,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101ab193f050002, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T14:43:47,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T14:43:47,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T14:43:47,980 WARN [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T14:43:47,981 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=bb7ea3e46d9c,44497,1733496226926 2024-12-06T14:43:47,983 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-06T14:43:47,985 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-06T14:43:47,994 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/hbase.id] with ID: bd0f430d-90fb-4709-894b-3c270cde836e 2024-12-06T14:43:47,994 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/.tmp/hbase.id 2024-12-06T14:43:47,999 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/hbase.id] 2024-12-06T14:43:48,044 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:48,049 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-06T14:43:48,067 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-06T14:43:48,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:48,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:48,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101ab193f050002, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:48,107 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T14:43:48,111 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-06T14:43:48,157 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:48,164 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T14:43:48,222 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store 2024-12-06T14:43:48,241 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-06T14:43:48,244 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:43:48,246 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T14:43:48,246 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:43:48,246 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:43:48,248 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T14:43:48,248 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:43:48,248 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:43:48,249 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733496228246Disabling compacts and flushes for region at 1733496228246Disabling writes for close at 1733496228248 (+2 ms)Writing region close event to WAL at 1733496228248Closed at 1733496228248 2024-12-06T14:43:48,249 WARN [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/.initializing 2024-12-06T14:43:48,249 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/WALs/bb7ea3e46d9c,44497,1733496226926 2024-12-06T14:43:48,260 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T14:43:48,273 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bb7ea3e46d9c%2C44497%2C1733496226926, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/WALs/bb7ea3e46d9c,44497,1733496226926, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/oldWALs, maxLogs=10 2024-12-06T14:43:48,307 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/WALs/bb7ea3e46d9c,44497,1733496226926/bb7ea3e46d9c%2C44497%2C1733496226926.1733496228277 2024-12-06T14:43:48,308 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T14:43:48,309 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-06T14:43:48,309 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:43:48,313 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:48,315 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:48,350 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:48,383 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-06T14:43:48,388 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:48,390 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:48,391 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:48,395 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-06T14:43:48,395 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:48,396 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:43:48,396 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:48,400 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-06T14:43:48,401 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:48,401 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:43:48,402 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:48,405 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-06T14:43:48,406 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:48,406 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:43:48,406 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:48,411 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:48,412 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:48,415 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:48,416 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:48,419 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T14:43:48,422 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:48,429 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T14:43:48,430 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74144477, jitterRate=0.10483880341053009}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T14:43:48,438 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733496228328Initializing all the Stores at 1733496228329 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496228329Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496228330 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496228331 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496228331Cleaning up temporary data from old regions at 1733496228416 (+85 ms)Region opened successfully at 1733496228438 (+22 ms) 2024-12-06T14:43:48,440 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-06T14:43:48,470 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604b4acd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bb7ea3e46d9c/172.17.0.2:0 2024-12-06T14:43:48,503 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-06T14:43:48,516 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-06T14:43:48,516 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-06T14:43:48,519 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-06T14:43:48,519 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-06T14:43:48,525 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-06T14:43:48,525 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-06T14:43:48,551 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-06T14:43:48,561 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-06T14:43:48,616 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-06T14:43:48,619 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-06T14:43:48,621 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-06T14:43:48,632 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-06T14:43:48,634 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-06T14:43:48,638 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-06T14:43:48,647 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-06T14:43:48,650 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-06T14:43:48,658 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-06T14:43:48,679 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-06T14:43:48,693 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-06T14:43:48,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T14:43:48,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:48,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101ab193f050002, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T14:43:48,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101ab193f050002, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:48,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T14:43:48,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:48,740 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=bb7ea3e46d9c,44497,1733496226926, sessionid=0x101ab193f050000, setting cluster-up flag (Was=false) 2024-12-06T14:43:48,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:48,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101ab193f050002, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:48,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:48,821 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-06T14:43:48,824 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bb7ea3e46d9c,44497,1733496226926 2024-12-06T14:43:48,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101ab193f050002, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:48,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:48,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:48,879 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-06T14:43:48,882 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bb7ea3e46d9c,44497,1733496226926 2024-12-06T14:43:48,887 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-06T14:43:48,903 INFO [RS:0;bb7ea3e46d9c:45909 {}] regionserver.HRegionServer(746): ClusterId : bd0f430d-90fb-4709-894b-3c270cde836e 2024-12-06T14:43:48,906 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T14:43:48,906 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HRegionServer(746): ClusterId : bd0f430d-90fb-4709-894b-3c270cde836e 2024-12-06T14:43:48,907 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T14:43:48,936 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T14:43:48,936 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T14:43:48,950 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T14:43:48,950 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T14:43:48,951 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T14:43:48,952 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@232c4806, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bb7ea3e46d9c/172.17.0.2:0 2024-12-06T14:43:48,965 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T14:43:48,966 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471144c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bb7ea3e46d9c/172.17.0.2:0 2024-12-06T14:43:48,976 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-06T14:43:48,984 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;bb7ea3e46d9c:45909 2024-12-06T14:43:48,988 INFO [RS:0;bb7ea3e46d9c:45909 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T14:43:48,989 INFO [RS:0;bb7ea3e46d9c:45909 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T14:43:48,989 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T14:43:48,990 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;bb7ea3e46d9c:42857 2024-12-06T14:43:48,990 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T14:43:48,990 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T14:43:48,991 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T14:43:48,992 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-06T14:43:48,992 INFO [RS:0;bb7ea3e46d9c:45909 {}] regionserver.HRegionServer(2659): reportForDuty to master=bb7ea3e46d9c,44497,1733496226926 with port=45909, startcode=1733496227736 2024-12-06T14:43:48,992 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HRegionServer(2659): reportForDuty to master=bb7ea3e46d9c,44497,1733496226926 with port=42857, startcode=1733496227864 2024-12-06T14:43:49,004 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-06T14:43:49,007 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T14:43:49,007 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T14:43:49,012 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: bb7ea3e46d9c,44497,1733496226926 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-06T14:43:49,021 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/bb7ea3e46d9c:0, corePoolSize=5, maxPoolSize=5 2024-12-06T14:43:49,022 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/bb7ea3e46d9c:0, corePoolSize=5, maxPoolSize=5 2024-12-06T14:43:49,022 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/bb7ea3e46d9c:0, corePoolSize=5, maxPoolSize=5 2024-12-06T14:43:49,022 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/bb7ea3e46d9c:0, corePoolSize=5, maxPoolSize=5 2024-12-06T14:43:49,023 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/bb7ea3e46d9c:0, corePoolSize=10, maxPoolSize=10 2024-12-06T14:43:49,023 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,023 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/bb7ea3e46d9c:0, corePoolSize=2, maxPoolSize=2 2024-12-06T14:43:49,023 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,039 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T14:43:49,040 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-06T14:43:49,048 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:49,049 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-06T14:43:49,057 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733496259057 2024-12-06T14:43:49,059 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-06T14:43:49,060 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-06T14:43:49,064 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-06T14:43:49,064 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-06T14:43:49,065 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-06T14:43:49,065 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-06T14:43:49,084 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,105 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-06T14:43:49,106 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-06T14:43:49,107 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-06T14:43:49,112 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-06T14:43:49,115 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-06T14:43:49,124 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.large.0-1733496229116,5,FailOnTimeoutGroup] 2024-12-06T14:43:49,131 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36125, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T14:43:49,132 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45973, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T14:43:49,142 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.small.0-1733496229133,5,FailOnTimeoutGroup] 2024-12-06T14:43:49,142 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,142 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-06T14:43:49,142 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44497 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T14:43:49,144 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,144 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,152 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-06T14:43:49,153 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f 2024-12-06T14:43:49,156 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44497 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bb7ea3e46d9c,45909,1733496227736 2024-12-06T14:43:49,160 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44497 {}] master.ServerManager(517): Registering regionserver=bb7ea3e46d9c,45909,1733496227736 2024-12-06T14:43:49,177 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f 2024-12-06T14:43:49,177 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-06T14:43:49,177 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T14:43:49,186 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-06T14:43:49,186 WARN [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-06T14:43:49,190 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:43:49,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T14:43:49,190 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] zookeeper.ZKUtil(111): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bb7ea3e46d9c,45909,1733496227736 2024-12-06T14:43:49,191 WARN [RS:0;bb7ea3e46d9c:45909 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T14:43:49,191 INFO [RS:0;bb7ea3e46d9c:45909 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T14:43:49,191 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,45909,1733496227736 2024-12-06T14:43:49,212 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bb7ea3e46d9c,45909,1733496227736] 2024-12-06T14:43:49,238 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-06T14:43:49,265 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T14:43:49,265 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:49,266 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:49,266 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-06T14:43:49,279 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T14:43:49,279 INFO [RS:0;bb7ea3e46d9c:45909 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T14:43:49,279 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:49,280 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:49,280 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-06T14:43:49,289 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HRegionServer(2659): reportForDuty to master=bb7ea3e46d9c,44497,1733496226926 with port=42857, startcode=1733496227864 2024-12-06T14:43:49,291 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44497 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bb7ea3e46d9c,42857,1733496227864 2024-12-06T14:43:49,291 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44497 {}] master.ServerManager(517): Registering regionserver=bb7ea3e46d9c,42857,1733496227864 2024-12-06T14:43:49,299 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f 2024-12-06T14:43:49,299 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-06T14:43:49,299 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T14:43:49,310 INFO [RS:0;bb7ea3e46d9c:45909 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T14:43:49,322 INFO [RS:0;bb7ea3e46d9c:45909 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T14:43:49,322 INFO [RS:0;bb7ea3e46d9c:45909 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,323 INFO [RS:0;bb7ea3e46d9c:45909 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T14:43:49,325 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T14:43:49,326 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:49,326 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:49,326 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-06T14:43:49,330 INFO [RS:0;bb7ea3e46d9c:45909 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T14:43:49,331 INFO [RS:0;bb7ea3e46d9c:45909 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,332 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,332 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,332 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,332 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,333 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,333 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bb7ea3e46d9c:0, corePoolSize=2, maxPoolSize=2 2024-12-06T14:43:49,335 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,335 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T14:43:49,335 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,335 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:49,335 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:49,335 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T14:43:49,337 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740 2024-12-06T14:43:49,337 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740 2024-12-06T14:43:49,337 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T14:43:49,337 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T14:43:49,338 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T14:43:49,338 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T14:43:49,344 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,344 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,345 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,345 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,345 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bb7ea3e46d9c:0, corePoolSize=3, maxPoolSize=3 2024-12-06T14:43:49,345 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0, corePoolSize=3, maxPoolSize=3 2024-12-06T14:43:49,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T14:43:49,358 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bb7ea3e46d9c,42857,1733496227864] 2024-12-06T14:43:49,359 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] zookeeper.ZKUtil(111): regionserver:42857-0x101ab193f050002, quorum=127.0.0.1:52310, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bb7ea3e46d9c,42857,1733496227864 2024-12-06T14:43:49,359 WARN [RS:1;bb7ea3e46d9c:42857 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T14:43:49,359 INFO [RS:1;bb7ea3e46d9c:42857 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T14:43:49,360 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,42857,1733496227864 2024-12-06T14:43:49,367 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T14:43:49,369 INFO [RS:0;bb7ea3e46d9c:45909 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,370 INFO [RS:0;bb7ea3e46d9c:45909 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,370 INFO [RS:0;bb7ea3e46d9c:45909 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,370 INFO [RS:0;bb7ea3e46d9c:45909 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,370 INFO [RS:0;bb7ea3e46d9c:45909 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,370 INFO [RS:0;bb7ea3e46d9c:45909 {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,45909,1733496227736-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T14:43:49,391 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T14:43:49,394 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T14:43:49,395 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75023694, jitterRate=0.11794015765190125}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T14:43:49,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733496229190Initializing all the Stores at 1733496229190Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496229191 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496229238 (+47 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496229238Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496229238Cleaning up temporary data from old regions at 1733496229337 (+99 ms)Region opened successfully at 1733496229397 (+60 ms) 2024-12-06T14:43:49,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T14:43:49,397 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T14:43:49,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T14:43:49,398 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T14:43:49,398 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T14:43:49,403 INFO [RS:0;bb7ea3e46d9c:45909 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T14:43:49,406 INFO [RS:1;bb7ea3e46d9c:42857 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T14:43:49,407 INFO [RS:1;bb7ea3e46d9c:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,407 INFO [RS:0;bb7ea3e46d9c:45909 {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,45909,1733496227736-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,407 INFO [RS:0;bb7ea3e46d9c:45909 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,407 INFO [RS:0;bb7ea3e46d9c:45909 {}] regionserver.Replication(171): bb7ea3e46d9c,45909,1733496227736 started 2024-12-06T14:43:49,413 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T14:43:49,414 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733496229397Disabling compacts and flushes for region at 1733496229397Disabling writes for close at 1733496229398 (+1 ms)Writing region close event to WAL at 1733496229412 (+14 ms)Closed at 1733496229413 (+1 ms) 2024-12-06T14:43:49,416 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T14:43:49,417 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-06T14:43:49,423 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T14:43:49,424 INFO [RS:1;bb7ea3e46d9c:42857 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T14:43:49,425 INFO [RS:1;bb7ea3e46d9c:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,425 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,425 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-06T14:43:49,425 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,426 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,426 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,426 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bb7ea3e46d9c:0, corePoolSize=2, maxPoolSize=2 2024-12-06T14:43:49,426 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,426 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,426 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,427 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,427 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,427 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:49,427 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bb7ea3e46d9c:0, corePoolSize=3, maxPoolSize=3 2024-12-06T14:43:49,427 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0, corePoolSize=3, maxPoolSize=3 2024-12-06T14:43:49,433 INFO [RS:0;bb7ea3e46d9c:45909 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,433 INFO [RS:0;bb7ea3e46d9c:45909 {}] regionserver.HRegionServer(1482): Serving as bb7ea3e46d9c,45909,1733496227736, RpcServer on bb7ea3e46d9c/172.17.0.2:45909, sessionid=0x101ab193f050001 2024-12-06T14:43:49,434 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T14:43:49,435 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bb7ea3e46d9c,45909,1733496227736 2024-12-06T14:43:49,435 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bb7ea3e46d9c,45909,1733496227736' 2024-12-06T14:43:49,435 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T14:43:49,436 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T14:43:49,437 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T14:43:49,437 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T14:43:49,437 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bb7ea3e46d9c,45909,1733496227736 2024-12-06T14:43:49,437 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bb7ea3e46d9c,45909,1733496227736' 2024-12-06T14:43:49,437 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T14:43:49,437 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T14:43:49,439 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T14:43:49,440 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-06T14:43:49,442 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T14:43:49,442 INFO [RS:0;bb7ea3e46d9c:45909 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T14:43:49,442 INFO [RS:0;bb7ea3e46d9c:45909 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T14:43:49,444 INFO [RS:1;bb7ea3e46d9c:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,444 INFO [RS:1;bb7ea3e46d9c:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,444 INFO [RS:1;bb7ea3e46d9c:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,444 INFO [RS:1;bb7ea3e46d9c:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,445 INFO [RS:1;bb7ea3e46d9c:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,445 INFO [RS:1;bb7ea3e46d9c:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,42857,1733496227864-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T14:43:49,470 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T14:43:49,470 INFO [RS:1;bb7ea3e46d9c:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,42857,1733496227864-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,470 INFO [RS:1;bb7ea3e46d9c:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,470 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.Replication(171): bb7ea3e46d9c,42857,1733496227864 started 2024-12-06T14:43:49,492 INFO [RS:1;bb7ea3e46d9c:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:49,493 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HRegionServer(1482): Serving as bb7ea3e46d9c,42857,1733496227864, RpcServer on bb7ea3e46d9c/172.17.0.2:42857, sessionid=0x101ab193f050002 2024-12-06T14:43:49,493 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T14:43:49,493 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bb7ea3e46d9c,42857,1733496227864 2024-12-06T14:43:49,493 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bb7ea3e46d9c,42857,1733496227864' 2024-12-06T14:43:49,494 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T14:43:49,494 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T14:43:49,495 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T14:43:49,495 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T14:43:49,495 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bb7ea3e46d9c,42857,1733496227864 2024-12-06T14:43:49,495 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bb7ea3e46d9c,42857,1733496227864' 2024-12-06T14:43:49,496 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T14:43:49,497 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T14:43:49,497 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T14:43:49,498 INFO [RS:1;bb7ea3e46d9c:42857 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T14:43:49,498 INFO [RS:1;bb7ea3e46d9c:42857 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T14:43:49,547 INFO [RS:0;bb7ea3e46d9c:45909 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T14:43:49,548 INFO [RS:0;bb7ea3e46d9c:45909 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bb7ea3e46d9c%2C45909%2C1733496227736, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,45909,1733496227736, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/oldWALs, maxLogs=55 2024-12-06T14:43:49,567 INFO [RS:0;bb7ea3e46d9c:45909 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,45909,1733496227736/bb7ea3e46d9c%2C45909%2C1733496227736.1733496229550 2024-12-06T14:43:49,567 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T14:43:49,593 DEBUG [bb7ea3e46d9c:44497 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-06T14:43:49,599 INFO [RS:1;bb7ea3e46d9c:42857 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T14:43:49,600 INFO [RS:1;bb7ea3e46d9c:42857 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bb7ea3e46d9c%2C42857%2C1733496227864, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,42857,1733496227864, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/oldWALs, maxLogs=55 2024-12-06T14:43:49,604 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=bb7ea3e46d9c,45909,1733496227736 2024-12-06T14:43:49,610 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bb7ea3e46d9c,45909,1733496227736, state=OPENING 2024-12-06T14:43:49,623 INFO [RS:1;bb7ea3e46d9c:42857 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,42857,1733496227864/bb7ea3e46d9c%2C42857%2C1733496227864.1733496229602 2024-12-06T14:43:49,624 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T14:43:49,632 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-06T14:43:49,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:49,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101ab193f050002, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:49,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:49,644 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:43:49,645 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:43:49,645 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:43:49,646 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T14:43:49,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=bb7ea3e46d9c,45909,1733496227736}] 2024-12-06T14:43:49,827 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T14:43:49,830 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46067, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T14:43:49,847 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-06T14:43:49,848 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T14:43:49,849 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-06T14:43:49,851 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bb7ea3e46d9c%2C45909%2C1733496227736.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,45909,1733496227736, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/oldWALs, maxLogs=55 2024-12-06T14:43:49,896 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,45909,1733496227736/bb7ea3e46d9c%2C45909%2C1733496227736.meta.1733496229853.meta 2024-12-06T14:43:49,899 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T14:43:49,899 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-06T14:43:49,901 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-06T14:43:49,904 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-06T14:43:49,908 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-06T14:43:49,913 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-06T14:43:49,914 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:43:49,914 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-06T14:43:49,914 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-06T14:43:49,916 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-06T14:43:49,917 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T14:43:49,917 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:49,918 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:49,919 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-06T14:43:49,921 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T14:43:49,921 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:49,921 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:49,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-06T14:43:49,923 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T14:43:49,923 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:49,923 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:49,923 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-06T14:43:49,925 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T14:43:49,925 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:49,925 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:49,925 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T14:43:49,926 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740 2024-12-06T14:43:49,927 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740 2024-12-06T14:43:49,927 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T14:43:49,928 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T14:43:49,928 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T14:43:49,929 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T14:43:49,930 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74058418, jitterRate=0.10355642437934875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T14:43:49,930 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-06T14:43:49,931 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733496229915Writing region info on filesystem at 1733496229915Initializing all the Stores at 1733496229915Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496229915Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496229916 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496229916Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496229916Cleaning up temporary data from old regions at 1733496229928 (+12 ms)Running coprocessor post-open hooks at 1733496229930 (+2 ms)Region opened successfully at 1733496229931 (+1 ms) 2024-12-06T14:43:49,936 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733496229817 2024-12-06T14:43:49,945 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-06T14:43:49,945 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-06T14:43:49,946 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,45909,1733496227736 2024-12-06T14:43:49,947 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bb7ea3e46d9c,45909,1733496227736, state=OPEN 2024-12-06T14:43:49,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T14:43:49,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101ab193f050002, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T14:43:49,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T14:43:49,992 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:43:49,992 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:43:49,992 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:43:49,993 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=bb7ea3e46d9c,45909,1733496227736 2024-12-06T14:43:49,996 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-06T14:43:49,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bb7ea3e46d9c,45909,1733496227736 in 345 msec 2024-12-06T14:43:50,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-06T14:43:50,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 574 msec 2024-12-06T14:43:50,004 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T14:43:50,004 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-06T14:43:50,027 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:50,028 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,45909,1733496227736, seqNum=-1] 2024-12-06T14:43:50,051 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:50,054 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59693, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:50,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1590 sec 2024-12-06T14:43:50,081 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733496230081, completionTime=-1 2024-12-06T14:43:50,084 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-06T14:43:50,085 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-06T14:43:50,114 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-06T14:43:50,114 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733496290114 2024-12-06T14:43:50,115 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733496350115 2024-12-06T14:43:50,115 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec 2024-12-06T14:43:50,118 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,44497,1733496226926-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:50,118 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,44497,1733496226926-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:50,119 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,44497,1733496226926-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:50,120 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-bb7ea3e46d9c:44497, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:50,120 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:50,121 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:50,126 DEBUG [master/bb7ea3e46d9c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-06T14:43:50,152 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.171sec 2024-12-06T14:43:50,154 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-06T14:43:50,155 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-06T14:43:50,157 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-06T14:43:50,157 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-06T14:43:50,158 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-06T14:43:50,159 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,44497,1733496226926-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T14:43:50,160 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,44497,1733496226926-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-06T14:43:50,171 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-06T14:43:50,172 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-06T14:43:50,173 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,44497,1733496226926-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:50,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:50,219 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-06T14:43:50,219 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-06T14:43:50,224 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,44497,-1 for getting cluster id 2024-12-06T14:43:50,227 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:50,235 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bd0f430d-90fb-4709-894b-3c270cde836e' 2024-12-06T14:43:50,238 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:50,238 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bd0f430d-90fb-4709-894b-3c270cde836e" 2024-12-06T14:43:50,239 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6377da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:50,239 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,44497,-1] 2024-12-06T14:43:50,242 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:50,244 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:50,245 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:50,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27097cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:50,250 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:50,259 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,45909,1733496227736, seqNum=-1] 2024-12-06T14:43:50,260 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:50,264 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:50,285 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=bb7ea3e46d9c,44497,1733496226926 2024-12-06T14:43:50,290 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T14:43:50,295 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is bb7ea3e46d9c,44497,1733496226926 2024-12-06T14:43:50,313 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5e597ded 2024-12-06T14:43:50,314 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T14:43:50,332 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46872, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T14:43:50,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44497 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733496230285', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T14:43:50,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44497 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733496230285 2024-12-06T14:43:50,352 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733496230285 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T14:43:50,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44497 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733496230285" procId is: 4 2024-12-06T14:43:50,355 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:50,357 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733496230285 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T14:43:50,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T14:43:50,381 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733496230285-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9e748811f71f7149bbeac36bc748022c, NAME => 'testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733496230285', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f 2024-12-06T14:43:50,395 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733496230285-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:43:50,395 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733496230285-pool-0 {}] regionserver.HRegion(1722): Closing 9e748811f71f7149bbeac36bc748022c, disabling compactions & flushes 2024-12-06T14:43:50,395 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733496230285-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:50,395 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733496230285-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:50,395 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733496230285-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. after waiting 0 ms 2024-12-06T14:43:50,395 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733496230285-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:50,396 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733496230285-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:50,396 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733496230285-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9e748811f71f7149bbeac36bc748022c: Waiting for close lock at 1733496230395Disabling compacts and flushes for region at 1733496230395Disabling writes for close at 1733496230395Writing region close event to WAL at 1733496230396 (+1 ms)Closed at 1733496230396 2024-12-06T14:43:50,397 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733496230285 execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T14:43:50,403 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733496230397"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733496230397"}]},"ts":"1733496230397"} 2024-12-06T14:43:50,407 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T14:43:50,408 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733496230285 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T14:43:50,411 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733496230285","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496230409"}]},"ts":"1733496230409"} 2024-12-06T14:43:50,416 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733496230285, state=ENABLING in hbase:meta 2024-12-06T14:43:50,421 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {bb7ea3e46d9c=0} racks are {/default-rack=0} 2024-12-06T14:43:50,423 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T14:43:50,423 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T14:43:50,423 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T14:43:50,423 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T14:43:50,423 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T14:43:50,423 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T14:43:50,423 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T14:43:50,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733496230285, region=9e748811f71f7149bbeac36bc748022c, ASSIGN}] 2024-12-06T14:43:50,426 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733496230285, region=9e748811f71f7149bbeac36bc748022c, ASSIGN 2024-12-06T14:43:50,428 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733496230285, region=9e748811f71f7149bbeac36bc748022c, ASSIGN; state=OFFLINE, location=bb7ea3e46d9c,45909,1733496227736; forceNewPlan=false, retain=false 2024-12-06T14:43:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T14:43:50,580 INFO [bb7ea3e46d9c:44497 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-06T14:43:50,581 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9e748811f71f7149bbeac36bc748022c, regionState=OPENING, regionLocation=bb7ea3e46d9c,45909,1733496227736 2024-12-06T14:43:50,585 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733496230285, region=9e748811f71f7149bbeac36bc748022c, ASSIGN because future has completed 2024-12-06T14:43:50,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e748811f71f7149bbeac36bc748022c, server=bb7ea3e46d9c,45909,1733496227736}] 2024-12-06T14:43:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T14:43:50,743 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:50,743 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9e748811f71f7149bbeac36bc748022c, NAME => 'testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c.', STARTKEY => '', ENDKEY => ''} 2024-12-06T14:43:50,744 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733496230285 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:50,744 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:43:50,744 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:50,744 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:50,745 INFO [StoreOpener-9e748811f71f7149bbeac36bc748022c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:50,750 INFO [StoreOpener-9e748811f71f7149bbeac36bc748022c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e748811f71f7149bbeac36bc748022c columnFamilyName family 2024-12-06T14:43:50,750 DEBUG [StoreOpener-9e748811f71f7149bbeac36bc748022c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:50,750 INFO [StoreOpener-9e748811f71f7149bbeac36bc748022c-1 {}] regionserver.HStore(327): Store=9e748811f71f7149bbeac36bc748022c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:43:50,751 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:50,752 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:50,752 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:50,752 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:50,752 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:50,753 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:50,760 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T14:43:50,760 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9e748811f71f7149bbeac36bc748022c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67686491, jitterRate=0.00860731303691864}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T14:43:50,761 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:50,761 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9e748811f71f7149bbeac36bc748022c: Running coprocessor pre-open hook at 1733496230744Writing region info on filesystem at 1733496230744Initializing all the Stores at 1733496230745 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496230745Cleaning up temporary data from old regions at 1733496230752 (+7 ms)Running coprocessor post-open hooks at 1733496230761 (+9 ms)Region opened successfully at 1733496230761 2024-12-06T14:43:50,763 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c., pid=6, masterSystemTime=1733496230740 2024-12-06T14:43:50,767 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9e748811f71f7149bbeac36bc748022c, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,45909,1733496227736 2024-12-06T14:43:50,767 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:50,767 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:50,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e748811f71f7149bbeac36bc748022c, server=bb7ea3e46d9c,45909,1733496227736 because future has completed 2024-12-06T14:43:50,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-06T14:43:50,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9e748811f71f7149bbeac36bc748022c, server=bb7ea3e46d9c,45909,1733496227736 in 186 msec 2024-12-06T14:43:50,777 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-06T14:43:50,777 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733496230285, region=9e748811f71f7149bbeac36bc748022c, ASSIGN in 350 msec 2024-12-06T14:43:50,778 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733496230285 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T14:43:50,778 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733496230285","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496230778"}]},"ts":"1733496230778"} 2024-12-06T14:43:50,781 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733496230285, state=ENABLED in hbase:meta 2024-12-06T14:43:50,782 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733496230285 execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T14:43:50,785 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733496230285 in 438 msec 2024-12-06T14:43:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T14:43:50,996 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733496230285 get assigned. Timeout = 60000ms 2024-12-06T14:43:50,996 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733496230285 completed 2024-12-06T14:43:50,999 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T14:43:51,006 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733496230285 assigned to meta. Checking AM states. 2024-12-06T14:43:51,007 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T14:43:51,007 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733496230285 assigned. 2024-12-06T14:43:51,016 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733496230285', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c., hostname=bb7ea3e46d9c,45909,1733496227736, seqNum=2] 2024-12-06T14:43:51,027 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9e748811f71f7149bbeac36bc748022c 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-06T14:43:51,056 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ae639e050a084f3abdb5e6837bb61896 2024-12-06T14:43:51,101 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c/.tmp/family/ae639e050a084f3abdb5e6837bb61896 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-06T14:43:51,110 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c/.tmp/family/ae639e050a084f3abdb5e6837bb61896 2024-12-06T14:43:51,154 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c/.tmp/family/ae639e050a084f3abdb5e6837bb61896 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c/family/ae639e050a084f3abdb5e6837bb61896 2024-12-06T14:43:51,156 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c/family/ae639e050a084f3abdb5e6837bb61896, delay=958 ms 2024-12-06T14:43:51,159 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c/family/ae639e050a084f3abdb5e6837bb61896, entries=2, sequenceid=6, filesize=4.9 K 2024-12-06T14:43:51,166 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 9e748811f71f7149bbeac36bc748022c in 137ms, sequenceid=6, compaction requested=false 2024-12-06T14:43:51,168 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733496230285' 2024-12-06T14:43:51,170 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9e748811f71f7149bbeac36bc748022c: 2024-12-06T14:43:52,116 DEBUG [hfile-prefetch-1733496231156 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key ae639e050a084f3abdb5e6837bb61896_0. 2024-12-06T14:43:52,120 DEBUG [hfile-prefetch-1733496231156 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key ae639e050a084f3abdb5e6837bb61896_117. 2024-12-06T14:43:52,121 DEBUG [hfile-prefetch-1733496231156 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c/family/ae639e050a084f3abdb5e6837bb61896, with total blocks 2, and data blocks 1 2024-12-06T14:43:52,121 DEBUG [hfile-prefetch-1733496231156 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-06T14:43:52,122 DEBUG [hfile-prefetch-1733496231156 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c/family/ae639e050a084f3abdb5e6837bb61896, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-06T14:43:52,122 DEBUG [hfile-prefetch-1733496231156 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c/family/ae639e050a084f3abdb5e6837bb61896. Will sleep 100ms and try the verification again. 2024-12-06T14:43:52,181 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44497 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=9e748811f71f7149bbeac36bc748022c, source=bb7ea3e46d9c,45909,1733496227736, destination=bb7ea3e46d9c,42857,1733496227864, warming up region on bb7ea3e46d9c,42857,1733496227864 2024-12-06T14:43:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44497 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T14:43:52,184 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44497 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=9e748811f71f7149bbeac36bc748022c, source=bb7ea3e46d9c,45909,1733496227736, destination=bb7ea3e46d9c,42857,1733496227864, running balancer 2024-12-06T14:43:52,185 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38471, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T14:43:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44497 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733496230285, region=9e748811f71f7149bbeac36bc748022c, REOPEN/MOVE 2024-12-06T14:43:52,185 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733496230285, region=9e748811f71f7149bbeac36bc748022c, REOPEN/MOVE 2024-12-06T14:43:52,186 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42857 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42857 {}] regionserver.HRegion(7855): Warmup {ENCODED => 9e748811f71f7149bbeac36bc748022c, NAME => 'testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c.', STARTKEY => '', ENDKEY => ''} 2024-12-06T14:43:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42857 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:43:52,189 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9e748811f71f7149bbeac36bc748022c, regionState=CLOSING, regionLocation=bb7ea3e46d9c,45909,1733496227736 2024-12-06T14:43:52,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733496230285, region=9e748811f71f7149bbeac36bc748022c, REOPEN/MOVE because future has completed 2024-12-06T14:43:52,195 INFO [StoreOpener-9e748811f71f7149bbeac36bc748022c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:52,196 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-06T14:43:52,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9e748811f71f7149bbeac36bc748022c, server=bb7ea3e46d9c,45909,1733496227736}] 2024-12-06T14:43:52,196 INFO [StoreOpener-9e748811f71f7149bbeac36bc748022c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e748811f71f7149bbeac36bc748022c columnFamilyName family 2024-12-06T14:43:52,196 DEBUG [StoreOpener-9e748811f71f7149bbeac36bc748022c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:52,202 DEBUG [StoreOpener-9e748811f71f7149bbeac36bc748022c-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c/family/ae639e050a084f3abdb5e6837bb61896 2024-12-06T14:43:52,202 INFO [StoreOpener-9e748811f71f7149bbeac36bc748022c-1 {}] regionserver.HStore(327): Store=9e748811f71f7149bbeac36bc748022c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:43:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42857 {}] regionserver.HRegion(1722): Closing 9e748811f71f7149bbeac36bc748022c, disabling compactions & flushes 2024-12-06T14:43:52,204 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42857 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42857 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42857 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. after waiting 0 ms 2024-12-06T14:43:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42857 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:52,206 DEBUG [StoreFileCloser-9e748811f71f7149bbeac36bc748022c-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c/family/ae639e050a084f3abdb5e6837bb61896 2024-12-06T14:43:52,206 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42857 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42857 {}] regionserver.HRegion(1676): Region close journal for 9e748811f71f7149bbeac36bc748022c: Waiting for close lock at 1733496232204Disabling compacts and flushes for region at 1733496232204Disabling writes for close at 1733496232204Writing region close event to WAL at 1733496232206 (+2 ms)Closed at 1733496232206 2024-12-06T14:43:52,208 WARN [hfile-prefetch-1733496231156 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c/family/ae639e050a084f3abdb5e6837bb61896, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-06T14:43:52,209 DEBUG [hfile-prefetch-1733496231156 {}] hfile.PrefetchExecutor(121): Prefetch completed for ae639e050a084f3abdb5e6837bb61896 2024-12-06T14:43:52,221 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-06T14:43:52,222 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-06T14:43:52,222 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T14:43:52,222 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T14:43:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44497 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-06T14:43:52,240 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-06T14:43:52,240 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733496232222 2024-12-06T14:43:52,240 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-06T14:43:52,364 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:52,365 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-06T14:43:52,366 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 9e748811f71f7149bbeac36bc748022c, disabling compactions & flushes 2024-12-06T14:43:52,366 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:52,367 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:52,367 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. after waiting 0 ms 2024-12-06T14:43:52,367 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:52,384 DEBUG [StoreFileCloser-9e748811f71f7149bbeac36bc748022c-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file ae639e050a084f3abdb5e6837bb61896, starting offset: 0, end offset: 9223372036854775807 2024-12-06T14:43:52,385 DEBUG [StoreFileCloser-9e748811f71f7149bbeac36bc748022c-family-1 {}] bucket.BucketCache(856): removed key ae639e050a084f3abdb5e6837bb61896_0 from back map with offset lock 2094080 in the evict process 2024-12-06T14:43:52,388 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-06T14:43:52,388 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-06T14:43:52,388 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T14:43:52,388 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T14:43:52,394 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-06T14:43:52,396 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:52,396 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 9e748811f71f7149bbeac36bc748022c: Waiting for close lock at 1733496232366Running coprocessor pre-close hooks at 1733496232366Disabling compacts and flushes for region at 1733496232366Disabling writes for close at 1733496232367 (+1 ms)Writing region close event to WAL at 1733496232386 (+19 ms)Running coprocessor post-close hooks at 1733496232394 (+8 ms)Closed at 1733496232395 (+1 ms) 2024-12-06T14:43:52,396 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 9e748811f71f7149bbeac36bc748022c move to bb7ea3e46d9c,42857,1733496227864 record at close sequenceid=6 2024-12-06T14:43:52,399 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:52,400 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9e748811f71f7149bbeac36bc748022c, regionState=CLOSED 2024-12-06T14:43:52,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9e748811f71f7149bbeac36bc748022c, server=bb7ea3e46d9c,45909,1733496227736 because future has completed 2024-12-06T14:43:52,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-06T14:43:52,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 9e748811f71f7149bbeac36bc748022c, server=bb7ea3e46d9c,45909,1733496227736 in 209 msec 2024-12-06T14:43:52,408 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733496230285, region=9e748811f71f7149bbeac36bc748022c, REOPEN/MOVE; state=CLOSED, location=bb7ea3e46d9c,42857,1733496227864; forceNewPlan=false, retain=false 2024-12-06T14:43:52,411 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-06T14:43:52,411 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733496232388 2024-12-06T14:43:52,412 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-06T14:43:52,559 INFO [bb7ea3e46d9c:44497 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-06T14:43:52,559 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9e748811f71f7149bbeac36bc748022c, regionState=OPENING, regionLocation=bb7ea3e46d9c,42857,1733496227864 2024-12-06T14:43:52,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733496230285, region=9e748811f71f7149bbeac36bc748022c, REOPEN/MOVE because future has completed 2024-12-06T14:43:52,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e748811f71f7149bbeac36bc748022c, server=bb7ea3e46d9c,42857,1733496227864}] 2024-12-06T14:43:52,721 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:52,721 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 9e748811f71f7149bbeac36bc748022c, NAME => 'testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c.', STARTKEY => '', ENDKEY => ''} 2024-12-06T14:43:52,722 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733496230285 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:52,722 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:43:52,722 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:52,722 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:52,728 INFO [StoreOpener-9e748811f71f7149bbeac36bc748022c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:52,730 INFO [StoreOpener-9e748811f71f7149bbeac36bc748022c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e748811f71f7149bbeac36bc748022c columnFamilyName family 2024-12-06T14:43:52,730 DEBUG [StoreOpener-9e748811f71f7149bbeac36bc748022c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:52,732 DEBUG [StoreFileOpener-9e748811f71f7149bbeac36bc748022c-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c/family/ae639e050a084f3abdb5e6837bb61896, delay=947 ms 2024-12-06T14:43:52,734 DEBUG [StoreOpener-9e748811f71f7149bbeac36bc748022c-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c/family/ae639e050a084f3abdb5e6837bb61896 2024-12-06T14:43:52,734 INFO [StoreOpener-9e748811f71f7149bbeac36bc748022c-1 {}] regionserver.HStore(327): Store=9e748811f71f7149bbeac36bc748022c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:43:52,734 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:52,736 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:52,736 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:52,736 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:52,737 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:52,742 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:52,743 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 9e748811f71f7149bbeac36bc748022c; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68145247, jitterRate=0.015443310141563416}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T14:43:52,743 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:52,745 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 9e748811f71f7149bbeac36bc748022c: Running coprocessor pre-open hook at 1733496232727Writing region info on filesystem at 1733496232727Initializing all the Stores at 1733496232727Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496232727Cleaning up temporary data from old regions at 1733496232737 (+10 ms)Running coprocessor post-open hooks at 1733496232743 (+6 ms)Region opened successfully at 1733496232745 (+2 ms) 2024-12-06T14:43:52,746 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c., pid=9, masterSystemTime=1733496232716 2024-12-06T14:43:52,749 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:52,749 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:52,750 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9e748811f71f7149bbeac36bc748022c, regionState=OPEN, openSeqNum=10, regionLocation=bb7ea3e46d9c,42857,1733496227864 2024-12-06T14:43:52,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e748811f71f7149bbeac36bc748022c, server=bb7ea3e46d9c,42857,1733496227864 because future has completed 2024-12-06T14:43:52,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-06T14:43:52,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 9e748811f71f7149bbeac36bc748022c, server=bb7ea3e46d9c,42857,1733496227864 in 193 msec 2024-12-06T14:43:52,760 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733496230285, region=9e748811f71f7149bbeac36bc748022c, REOPEN/MOVE in 574 msec 2024-12-06T14:43:52,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-06T14:43:52,788 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T14:43:52,788 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T14:43:52,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:52,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:52,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:52,795 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-06T14:43:52,795 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-12-06T14:43:52,796 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=bb7ea3e46d9c,44497,1733496226926 2024-12-06T14:43:52,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T14:43:52,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:52,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101ab193f050002, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T14:43:52,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101ab193f050002, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:52,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T14:43:52,817 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T14:43:52,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:52,817 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T14:43:52,817 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42857-0x101ab193f050002, quorum=127.0.0.1:52310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T14:43:52,817 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T14:43:52,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:52,818 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bb7ea3e46d9c,45909,1733496227736' ***** 2024-12-06T14:43:52,818 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-06T14:43:52,818 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bb7ea3e46d9c,42857,1733496227864' ***** 2024-12-06T14:43:52,818 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-06T14:43:52,818 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T14:43:52,819 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T14:43:52,819 INFO [RS:1;bb7ea3e46d9c:42857 {}] bucket.BucketCache(1738): Disabling cache 2024-12-06T14:43:52,819 INFO [RS:0;bb7ea3e46d9c:45909 {}] bucket.BucketCache(1738): Disabling cache 2024-12-06T14:43:52,819 INFO [RS:1;bb7ea3e46d9c:42857 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/bucket.persistence 2024-12-06T14:43:52,819 INFO [RS:0;bb7ea3e46d9c:45909 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/bucket.persistence 2024-12-06T14:43:52,819 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-06T14:43:52,819 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-06T14:43:52,820 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-06T14:43:52,820 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-06T14:43:52,820 INFO [RS:0;bb7ea3e46d9c:45909 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-06T14:43:52,820 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-06T14:43:52,820 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-06T14:43:52,820 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-06T14:43:52,821 INFO [RS:1;bb7ea3e46d9c:42857 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-06T14:43:52,821 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-06T14:43:52,821 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] bucket.BucketCache(1487): Thread RS:1;bb7ea3e46d9c:42857 started persisting bucket cache to file 2024-12-06T14:43:52,821 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T14:43:52,821 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T14:43:52,826 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-06T14:43:52,826 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733496232821 2024-12-06T14:43:52,826 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] bucket.BucketCache(1509): Thread RS:1;bb7ea3e46d9c:42857 finished persisting bucket cache to file, renaming 2024-12-06T14:43:52,827 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T14:43:52,827 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T14:43:52,827 INFO [RS:1;bb7ea3e46d9c:42857 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T14:43:52,827 INFO [RS:1;bb7ea3e46d9c:42857 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T14:43:52,827 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HRegionServer(3091): Received CLOSE for 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:52,829 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HRegionServer(959): stopping server bb7ea3e46d9c,42857,1733496227864 2024-12-06T14:43:52,829 INFO [RS:1;bb7ea3e46d9c:42857 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T14:43:52,829 INFO [RS:1;bb7ea3e46d9c:42857 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;bb7ea3e46d9c:42857. 2024-12-06T14:43:52,829 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T14:43:52,829 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:52,829 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-06T14:43:52,830 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9e748811f71f7149bbeac36bc748022c, disabling compactions & flushes 2024-12-06T14:43:52,830 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:52,830 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:52,830 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. after waiting 0 ms 2024-12-06T14:43:52,830 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:52,830 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] bucket.BucketCache(1487): Thread RS:0;bb7ea3e46d9c:45909 started persisting bucket cache to file 2024-12-06T14:43:52,831 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-06T14:43:52,831 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HRegionServer(1325): Online Regions={9e748811f71f7149bbeac36bc748022c=testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c.} 2024-12-06T14:43:52,831 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T14:43:52,831 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T14:43:52,831 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HRegionServer(1351): Waiting on 9e748811f71f7149bbeac36bc748022c 2024-12-06T14:43:52,838 DEBUG [StoreFileCloser-9e748811f71f7149bbeac36bc748022c-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c/family/ae639e050a084f3abdb5e6837bb61896 2024-12-06T14:43:52,839 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-06T14:43:52,839 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733496232831 2024-12-06T14:43:52,839 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] bucket.BucketCache(1509): Thread RS:0;bb7ea3e46d9c:45909 finished persisting bucket cache to file, renaming 2024-12-06T14:43:52,839 INFO [RS:0;bb7ea3e46d9c:45909 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T14:43:52,839 INFO [RS:0;bb7ea3e46d9c:45909 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T14:43:52,839 INFO [RS:0;bb7ea3e46d9c:45909 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T14:43:52,840 INFO [RS:0;bb7ea3e46d9c:45909 {}] regionserver.HRegionServer(959): stopping server bb7ea3e46d9c,45909,1733496227736 2024-12-06T14:43:52,840 INFO [RS:0;bb7ea3e46d9c:45909 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T14:43:52,840 INFO [RS:0;bb7ea3e46d9c:45909 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;bb7ea3e46d9c:45909. 2024-12-06T14:43:52,840 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T14:43:52,840 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T14:43:52,840 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:52,840 INFO [RS:0;bb7ea3e46d9c:45909 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T14:43:52,840 INFO [RS:0;bb7ea3e46d9c:45909 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T14:43:52,840 INFO [RS:0;bb7ea3e46d9c:45909 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T14:43:52,840 INFO [RS:0;bb7ea3e46d9c:45909 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-06T14:43:52,842 INFO [RS:0;bb7ea3e46d9c:45909 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-06T14:43:52,842 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-06T14:43:52,842 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T14:43:52,842 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-06T14:43:52,842 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T14:43:52,843 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T14:43:52,843 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T14:43:52,843 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T14:43:52,843 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T14:43:52,844 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-06T14:43:52,851 INFO [regionserver/bb7ea3e46d9c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T14:43:52,852 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnRegionMove1733496230285/9e748811f71f7149bbeac36bc748022c/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-06T14:43:52,853 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:52,853 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9e748811f71f7149bbeac36bc748022c: Waiting for close lock at 1733496232829Running coprocessor pre-close hooks at 1733496232830 (+1 ms)Disabling compacts and flushes for region at 1733496232830Disabling writes for close at 1733496232830Writing region close event to WAL at 1733496232840 (+10 ms)Running coprocessor post-close hooks at 1733496232853 (+13 ms)Closed at 1733496232853 2024-12-06T14:43:52,853 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c. 2024-12-06T14:43:52,871 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 11c61f7da5f24a35955cd473d246ba6a 2024-12-06T14:43:52,879 INFO [regionserver/bb7ea3e46d9c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T14:43:52,896 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/info/11c61f7da5f24a35955cd473d246ba6a is 205, key is testBlockEvictionOnRegionMove1733496230285,,1733496230336.9e748811f71f7149bbeac36bc748022c./info:regioninfo/1733496232750/Put/seqid=0 2024-12-06T14:43:52,897 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/info/11c61f7da5f24a35955cd473d246ba6a 2024-12-06T14:43:52,913 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d173d97e5ef046969919f32ce14568ed 2024-12-06T14:43:52,932 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/ns/d173d97e5ef046969919f32ce14568ed is 43, key is default/ns:d/1733496230063/Put/seqid=0 2024-12-06T14:43:52,933 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/ns/d173d97e5ef046969919f32ce14568ed 2024-12-06T14:43:52,943 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ee37ceb5538b4f8c884b0764cef29efd 2024-12-06T14:43:52,961 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/table/ee37ceb5538b4f8c884b0764cef29efd is 78, key is testBlockEvictionOnRegionMove1733496230285/table:state/1733496230778/Put/seqid=0 2024-12-06T14:43:52,962 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/table/ee37ceb5538b4f8c884b0764cef29efd 2024-12-06T14:43:52,966 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/info/11c61f7da5f24a35955cd473d246ba6a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/info/11c61f7da5f24a35955cd473d246ba6a 2024-12-06T14:43:52,968 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/info/11c61f7da5f24a35955cd473d246ba6a, delay=937 ms 2024-12-06T14:43:52,972 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/info/11c61f7da5f24a35955cd473d246ba6a, entries=15, sequenceid=15, filesize=7.6 K 2024-12-06T14:43:52,973 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/ns/d173d97e5ef046969919f32ce14568ed as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/ns/d173d97e5ef046969919f32ce14568ed 2024-12-06T14:43:52,974 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/ns/d173d97e5ef046969919f32ce14568ed, delay=952 ms 2024-12-06T14:43:52,978 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/ns/d173d97e5ef046969919f32ce14568ed, entries=2, sequenceid=15, filesize=5.0 K 2024-12-06T14:43:52,978 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/table/ee37ceb5538b4f8c884b0764cef29efd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/table/ee37ceb5538b4f8c884b0764cef29efd 2024-12-06T14:43:52,980 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/table/ee37ceb5538b4f8c884b0764cef29efd, delay=900 ms 2024-12-06T14:43:52,982 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/table/ee37ceb5538b4f8c884b0764cef29efd, entries=2, sequenceid=15, filesize=5.3 K 2024-12-06T14:43:52,983 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 140ms, sequenceid=15, compaction requested=false 2024-12-06T14:43:52,983 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-06T14:43:52,993 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/info/11c61f7da5f24a35955cd473d246ba6a 2024-12-06T14:43:52,996 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/ns/d173d97e5ef046969919f32ce14568ed 2024-12-06T14:43:53,000 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/table/ee37ceb5538b4f8c884b0764cef29efd 2024-12-06T14:43:53,006 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-06T14:43:53,007 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T14:43:53,007 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T14:43:53,008 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733496232842Running coprocessor pre-close hooks at 1733496232842Disabling compacts and flushes for region at 1733496232842Disabling writes for close at 1733496232843 (+1 ms)Obtaining lock to block concurrent updates at 1733496232844 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733496232844Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733496232844Flushing stores of hbase:meta,,1.1588230740 at 1733496232845 (+1 ms)Flushing 1588230740/info: creating writer at 1733496232845Flushing 1588230740/info: appending metadata at 1733496232894 (+49 ms)Flushing 1588230740/info: closing flushed file at 1733496232894Flushing 1588230740/ns: creating writer at 1733496232901 (+7 ms)Flushing 1588230740/ns: appending metadata at 1733496232930 (+29 ms)Flushing 1588230740/ns: closing flushed file at 1733496232930Flushing 1588230740/table: creating writer at 1733496232937 (+7 ms)Flushing 1588230740/table: appending metadata at 1733496232960 (+23 ms)Flushing 1588230740/table: closing flushed file at 1733496232960Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ee3bb38: reopening flushed file at 1733496232965 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bbd823b: reopening flushed file at 1733496232972 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24d7ebfd: reopening flushed file at 1733496232978 (+6 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 140ms, sequenceid=15, compaction requested=false at 1733496232983 (+5 ms)Writing region close event to WAL at 1733496233001 (+18 ms)Running coprocessor post-close hooks at 1733496233007 (+6 ms)Closed at 1733496233007 2024-12-06T14:43:53,008 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-06T14:43:53,031 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HRegionServer(976): stopping server bb7ea3e46d9c,42857,1733496227864; all regions closed. 2024-12-06T14:43:53,035 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/oldWALs 2024-12-06T14:43:53,036 INFO [RS:1;bb7ea3e46d9c:42857 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bb7ea3e46d9c%2C42857%2C1733496227864:(num 1733496229602) 2024-12-06T14:43:53,036 DEBUG [RS:1;bb7ea3e46d9c:42857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:53,036 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T14:43:53,036 INFO [RS:1;bb7ea3e46d9c:42857 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T14:43:53,036 INFO [RS:1;bb7ea3e46d9c:42857 {}] hbase.ChoreService(370): Chore service for: regionserver/bb7ea3e46d9c:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-06T14:43:53,036 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T14:43:53,036 INFO [regionserver/bb7ea3e46d9c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T14:43:53,036 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T14:43:53,037 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T14:43:53,037 INFO [RS:1;bb7ea3e46d9c:42857 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T14:43:53,037 INFO [RS:1;bb7ea3e46d9c:42857 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42857 2024-12-06T14:43:53,042 INFO [RS:0;bb7ea3e46d9c:45909 {}] regionserver.HRegionServer(976): stopping server bb7ea3e46d9c,45909,1733496227736; all regions closed. 2024-12-06T14:43:53,045 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/oldWALs 2024-12-06T14:43:53,045 INFO [RS:0;bb7ea3e46d9c:45909 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bb7ea3e46d9c%2C45909%2C1733496227736.meta:.meta(num 1733496229853) 2024-12-06T14:43:53,047 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/oldWALs 2024-12-06T14:43:53,047 INFO [RS:0;bb7ea3e46d9c:45909 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bb7ea3e46d9c%2C45909%2C1733496227736:(num 1733496229550) 2024-12-06T14:43:53,047 DEBUG [RS:0;bb7ea3e46d9c:45909 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:53,048 INFO [RS:0;bb7ea3e46d9c:45909 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T14:43:53,048 INFO [RS:0;bb7ea3e46d9c:45909 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T14:43:53,048 INFO [RS:0;bb7ea3e46d9c:45909 {}] hbase.ChoreService(370): Chore service for: regionserver/bb7ea3e46d9c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T14:43:53,048 INFO [RS:0;bb7ea3e46d9c:45909 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T14:43:53,048 INFO [RS:0;bb7ea3e46d9c:45909 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45909 2024-12-06T14:43:53,049 INFO [regionserver/bb7ea3e46d9c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T14:43:53,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T14:43:53,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101ab193f050002, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bb7ea3e46d9c,42857,1733496227864 2024-12-06T14:43:53,058 INFO [RS:1;bb7ea3e46d9c:42857 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T14:43:53,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bb7ea3e46d9c,45909,1733496227736 2024-12-06T14:43:53,069 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bb7ea3e46d9c,45909,1733496227736] 2024-12-06T14:43:53,069 INFO [RS:0;bb7ea3e46d9c:45909 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T14:43:53,089 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bb7ea3e46d9c,45909,1733496227736 already deleted, retry=false 2024-12-06T14:43:53,090 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bb7ea3e46d9c,45909,1733496227736 expired; onlineServers=1 2024-12-06T14:43:53,090 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bb7ea3e46d9c,42857,1733496227864] 2024-12-06T14:43:53,099 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bb7ea3e46d9c,42857,1733496227864 already deleted, retry=false 2024-12-06T14:43:53,100 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bb7ea3e46d9c,42857,1733496227864 expired; onlineServers=0 2024-12-06T14:43:53,100 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'bb7ea3e46d9c,44497,1733496226926' ***** 2024-12-06T14:43:53,100 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-06T14:43:53,100 INFO [M:0;bb7ea3e46d9c:44497 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T14:43:53,100 INFO [M:0;bb7ea3e46d9c:44497 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T14:43:53,100 DEBUG [M:0;bb7ea3e46d9c:44497 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-06T14:43:53,101 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-06T14:43:53,101 DEBUG [M:0;bb7ea3e46d9c:44497 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-06T14:43:53,101 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.small.0-1733496229133 {}] cleaner.HFileCleaner(306): Exit Thread[master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.small.0-1733496229133,5,FailOnTimeoutGroup] 2024-12-06T14:43:53,101 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.large.0-1733496229116 {}] cleaner.HFileCleaner(306): Exit Thread[master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.large.0-1733496229116,5,FailOnTimeoutGroup] 2024-12-06T14:43:53,101 INFO [M:0;bb7ea3e46d9c:44497 {}] hbase.ChoreService(370): Chore service for: master/bb7ea3e46d9c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-06T14:43:53,101 INFO [M:0;bb7ea3e46d9c:44497 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T14:43:53,101 DEBUG [M:0;bb7ea3e46d9c:44497 {}] master.HMaster(1795): Stopping service threads 2024-12-06T14:43:53,101 INFO [M:0;bb7ea3e46d9c:44497 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-06T14:43:53,101 INFO [M:0;bb7ea3e46d9c:44497 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T14:43:53,102 INFO [M:0;bb7ea3e46d9c:44497 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-06T14:43:53,102 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-06T14:43:53,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T14:43:53,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:53,110 DEBUG [M:0;bb7ea3e46d9c:44497 {}] zookeeper.ZKUtil(347): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-06T14:43:53,110 WARN [M:0;bb7ea3e46d9c:44497 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-06T14:43:53,111 INFO [M:0;bb7ea3e46d9c:44497 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/.lastflushedseqids 2024-12-06T14:43:53,120 INFO [M:0;bb7ea3e46d9c:44497 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-06T14:43:53,120 INFO [M:0;bb7ea3e46d9c:44497 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-06T14:43:53,120 DEBUG [M:0;bb7ea3e46d9c:44497 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T14:43:53,121 INFO [M:0;bb7ea3e46d9c:44497 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:43:53,121 DEBUG [M:0;bb7ea3e46d9c:44497 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:43:53,121 DEBUG [M:0;bb7ea3e46d9c:44497 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T14:43:53,121 DEBUG [M:0;bb7ea3e46d9c:44497 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:43:53,121 INFO [M:0;bb7ea3e46d9c:44497 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-12-06T14:43:53,131 DEBUG [M:0;bb7ea3e46d9c:44497 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6884da6d671048c5871050ed0ac58b0c 2024-12-06T14:43:53,145 DEBUG [M:0;bb7ea3e46d9c:44497 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6884da6d671048c5871050ed0ac58b0c is 82, key is hbase:meta,,1/info:regioninfo/1733496229946/Put/seqid=0 2024-12-06T14:43:53,146 INFO [M:0;bb7ea3e46d9c:44497 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6884da6d671048c5871050ed0ac58b0c 2024-12-06T14:43:53,155 DEBUG [M:0;bb7ea3e46d9c:44497 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fb23b136bdb4458eadf3a39a9e0f038d 2024-12-06T14:43:53,169 DEBUG [M:0;bb7ea3e46d9c:44497 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fb23b136bdb4458eadf3a39a9e0f038d is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733496230784/Put/seqid=0 2024-12-06T14:43:53,169 INFO [M:0;bb7ea3e46d9c:44497 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fb23b136bdb4458eadf3a39a9e0f038d 2024-12-06T14:43:53,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101ab193f050002, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T14:43:53,179 DEBUG [M:0;bb7ea3e46d9c:44497 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 463f9c30d63d4a0e8edaebfe69075126 2024-12-06T14:43:53,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101ab193f050002, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T14:43:53,180 INFO [RS:1;bb7ea3e46d9c:42857 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T14:43:53,180 INFO [RS:1;bb7ea3e46d9c:42857 {}] regionserver.HRegionServer(1031): Exiting; stopping=bb7ea3e46d9c,42857,1733496227864; zookeeper connection closed. 2024-12-06T14:43:53,182 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@403f54fa {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@403f54fa 2024-12-06T14:43:53,189 INFO [RS:0;bb7ea3e46d9c:45909 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T14:43:53,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T14:43:53,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45909-0x101ab193f050001, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T14:43:53,189 INFO [RS:0;bb7ea3e46d9c:45909 {}] regionserver.HRegionServer(1031): Exiting; stopping=bb7ea3e46d9c,45909,1733496227736; zookeeper connection closed. 2024-12-06T14:43:53,190 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@389c7118 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@389c7118 2024-12-06T14:43:53,190 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-06T14:43:53,196 DEBUG [M:0;bb7ea3e46d9c:44497 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/463f9c30d63d4a0e8edaebfe69075126 is 69, key is bb7ea3e46d9c,42857,1733496227864/rs:state/1733496229291/Put/seqid=0 2024-12-06T14:43:53,201 INFO [M:0;bb7ea3e46d9c:44497 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/463f9c30d63d4a0e8edaebfe69075126 2024-12-06T14:43:53,203 DEBUG [M:0;bb7ea3e46d9c:44497 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6884da6d671048c5871050ed0ac58b0c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6884da6d671048c5871050ed0ac58b0c 2024-12-06T14:43:53,205 INFO [M:0;bb7ea3e46d9c:44497 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6884da6d671048c5871050ed0ac58b0c, entries=8, sequenceid=84, filesize=5.5 K 2024-12-06T14:43:53,206 DEBUG [M:0;bb7ea3e46d9c:44497 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fb23b136bdb4458eadf3a39a9e0f038d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fb23b136bdb4458eadf3a39a9e0f038d 2024-12-06T14:43:53,207 INFO [M:0;bb7ea3e46d9c:44497 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fb23b136bdb4458eadf3a39a9e0f038d, entries=9, sequenceid=84, filesize=6.5 K 2024-12-06T14:43:53,208 DEBUG [M:0;bb7ea3e46d9c:44497 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/463f9c30d63d4a0e8edaebfe69075126 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/463f9c30d63d4a0e8edaebfe69075126 2024-12-06T14:43:53,210 INFO [M:0;bb7ea3e46d9c:44497 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/463f9c30d63d4a0e8edaebfe69075126, entries=2, sequenceid=84, filesize=5.1 K 2024-12-06T14:43:53,211 INFO [M:0;bb7ea3e46d9c:44497 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 90ms, sequenceid=84, compaction requested=false 2024-12-06T14:43:53,221 INFO [M:0;bb7ea3e46d9c:44497 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:43:53,222 DEBUG [M:0;bb7ea3e46d9c:44497 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733496233120Disabling compacts and flushes for region at 1733496233120Disabling writes for close at 1733496233121 (+1 ms)Obtaining lock to block concurrent updates at 1733496233121Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733496233121Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1733496233121Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733496233122 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733496233122Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733496233144 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733496233144Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733496233149 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733496233168 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733496233168Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733496233172 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733496233195 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733496233195Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3eacd68d: reopening flushed file at 1733496233203 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f172e7d: reopening flushed file at 1733496233206 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@131b1e54: reopening flushed file at 1733496233208 (+2 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 90ms, sequenceid=84, compaction requested=false at 1733496233211 (+3 ms)Writing region close event to WAL at 1733496233221 (+10 ms)Closed at 1733496233221 2024-12-06T14:43:53,226 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T14:43:53,226 INFO [M:0;bb7ea3e46d9c:44497 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-06T14:43:53,226 INFO [M:0;bb7ea3e46d9c:44497 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44497 2024-12-06T14:43:53,226 INFO [M:0;bb7ea3e46d9c:44497 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T14:43:53,338 INFO [M:0;bb7ea3e46d9c:44497 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T14:43:53,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T14:43:53,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44497-0x101ab193f050000, quorum=127.0.0.1:52310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T14:43:53,343 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-06T14:43:53,363 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-06T14:43:53,364 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:53,373 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/bb7ea3e46d9c:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733496232974 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733496231156 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/bb7ea3e46d9c:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: regionserver/bb7ea3e46d9c:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733496232968 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733496232732 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/bb7ea3e46d9c:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=586 (was 577) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6479 (was 6849) 2024-12-06T14:43:53,380 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=586, ProcessCount=11, AvailableMemoryMB=6479 2024-12-06T14:43:53,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/hadoop-log-dir so I do NOT create it in target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30 2024-12-06T14:43:53,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/hadoop-tmp-dir so I do NOT create it in target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30 2024-12-06T14:43:53,381 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:53,390 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30/cluster_e4c7bf86-7914-fd3c-137a-c81cbd812f93, deleteOnExit=true 2024-12-06T14:43:53,399 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30/cluster_e4c7bf86-7914-fd3c-137a-c81cbd812f93/zookeeper_0, clientPort=59695, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30/cluster_e4c7bf86-7914-fd3c-137a-c81cbd812f93/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30/cluster_e4c7bf86-7914-fd3c-137a-c81cbd812f93/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-06T14:43:53,401 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59695 2024-12-06T14:43:53,414 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f with version=8 2024-12-06T14:43:53,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/hbase-staging 2024-12-06T14:43:53,416 INFO [Time-limited test {}] client.ConnectionUtils(128): master/bb7ea3e46d9c:0 server-side Connection retries=45 2024-12-06T14:43:53,416 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:53,417 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:53,417 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T14:43:53,417 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:53,417 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T14:43:53,417 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T14:43:53,417 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T14:43:53,418 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42883 2024-12-06T14:43:53,419 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42883 connecting to ZooKeeper ensemble=127.0.0.1:59695 2024-12-06T14:43:53,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:428830x0, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T14:43:53,501 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42883-0x101ab19595c0000 connected 2024-12-06T14:43:53,585 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:53,585 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:53,586 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T14:43:53,586 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f, hbase.cluster.distributed=false 2024-12-06T14:43:53,588 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T14:43:53,589 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42883 2024-12-06T14:43:53,589 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42883 2024-12-06T14:43:53,590 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42883 2024-12-06T14:43:53,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42883 2024-12-06T14:43:53,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42883 2024-12-06T14:43:53,612 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bb7ea3e46d9c:0 server-side Connection retries=45 2024-12-06T14:43:53,613 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:53,613 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:53,613 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T14:43:53,613 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:53,613 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T14:43:53,613 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T14:43:53,613 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T14:43:53,614 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41829 2024-12-06T14:43:53,616 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41829 connecting to ZooKeeper ensemble=127.0.0.1:59695 2024-12-06T14:43:53,616 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:53,616 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:53,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:418290x0, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T14:43:53,628 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:418290x0, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T14:43:53,628 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41829-0x101ab19595c0001 connected 2024-12-06T14:43:53,628 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T14:43:53,630 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30/bucket.cache 2024-12-06T14:43:53,630 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-06T14:43:53,630 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-06T14:43:53,630 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-06T14:43:53,630 INFO [Thread-118 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-06T14:43:53,631 WARN [Thread-118 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-06T14:43:53,631 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-06T14:43:53,631 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-06T14:43:53,631 INFO [Thread-118 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30/bucket.persistence is complete. 2024-12-06T14:43:53,631 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T14:43:53,632 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T14:43:53,633 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T14:43:53,640 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41829 2024-12-06T14:43:53,641 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41829 2024-12-06T14:43:53,643 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41829 2024-12-06T14:43:53,644 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41829 2024-12-06T14:43:53,645 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41829 2024-12-06T14:43:53,660 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bb7ea3e46d9c:0 server-side Connection retries=45 2024-12-06T14:43:53,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:53,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:53,660 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T14:43:53,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:53,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T14:43:53,660 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T14:43:53,660 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T14:43:53,662 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41371 2024-12-06T14:43:53,663 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41371 connecting to ZooKeeper ensemble=127.0.0.1:59695 2024-12-06T14:43:53,663 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:53,663 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:53,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:413710x0, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T14:43:53,675 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:413710x0, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T14:43:53,675 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41371-0x101ab19595c0002 connected 2024-12-06T14:43:53,676 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T14:43:53,678 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30/bucket.cache 2024-12-06T14:43:53,678 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-06T14:43:53,679 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-06T14:43:53,679 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-06T14:43:53,682 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-06T14:43:53,682 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-06T14:43:53,682 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-06T14:43:53,682 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30/bucket.persistence is complete. 2024-12-06T14:43:53,683 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-06T14:43:53,687 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T14:43:53,689 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41371-0x101ab19595c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T14:43:53,691 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41371-0x101ab19595c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T14:43:53,700 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41371 2024-12-06T14:43:53,702 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41371 2024-12-06T14:43:53,704 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41371 2024-12-06T14:43:53,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41371 2024-12-06T14:43:53,708 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41371 2024-12-06T14:43:53,722 DEBUG [M:0;bb7ea3e46d9c:42883 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;bb7ea3e46d9c:42883 2024-12-06T14:43:53,724 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/bb7ea3e46d9c,42883,1733496233416 2024-12-06T14:43:53,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T14:43:53,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41371-0x101ab19595c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T14:43:53,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T14:43:53,734 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/bb7ea3e46d9c,42883,1733496233416 2024-12-06T14:43:53,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41371-0x101ab19595c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T14:43:53,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T14:43:53,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:53,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:53,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41371-0x101ab19595c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:53,744 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T14:43:53,745 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/bb7ea3e46d9c,42883,1733496233416 from backup master directory 2024-12-06T14:43:53,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41371-0x101ab19595c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T14:43:53,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/bb7ea3e46d9c,42883,1733496233416 2024-12-06T14:43:53,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T14:43:53,753 WARN [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T14:43:53,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T14:43:53,753 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=bb7ea3e46d9c,42883,1733496233416 2024-12-06T14:43:53,755 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/hbase.id] with ID: 3baa005d-7704-4d42-9cf6-fd7613fbd907 2024-12-06T14:43:53,755 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/.tmp/hbase.id 2024-12-06T14:43:53,761 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/hbase.id] 2024-12-06T14:43:53,804 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:53,804 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-06T14:43:53,806 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-12-06T14:43:53,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:53,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41371-0x101ab19595c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:53,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:53,822 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T14:43:53,823 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-06T14:43:53,823 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T14:43:53,853 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store 2024-12-06T14:43:53,862 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:43:53,862 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T14:43:53,862 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:43:53,862 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:43:53,862 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T14:43:53,862 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:43:53,862 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:43:53,862 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733496233862Disabling compacts and flushes for region at 1733496233862Disabling writes for close at 1733496233862Writing region close event to WAL at 1733496233862Closed at 1733496233862 2024-12-06T14:43:53,863 WARN [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/.initializing 2024-12-06T14:43:53,863 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/WALs/bb7ea3e46d9c,42883,1733496233416 2024-12-06T14:43:53,871 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T14:43:53,875 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bb7ea3e46d9c%2C42883%2C1733496233416, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/WALs/bb7ea3e46d9c,42883,1733496233416, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/oldWALs, maxLogs=10 2024-12-06T14:43:53,898 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/WALs/bb7ea3e46d9c,42883,1733496233416/bb7ea3e46d9c%2C42883%2C1733496233416.1733496233875 2024-12-06T14:43:53,901 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T14:43:53,901 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-06T14:43:53,902 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:43:53,902 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:53,902 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:53,903 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:53,910 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-06T14:43:53,910 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:53,911 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:53,911 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:53,922 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-06T14:43:53,923 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:53,923 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:43:53,923 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:53,928 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-06T14:43:53,928 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:53,928 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:43:53,928 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:53,931 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-06T14:43:53,932 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:53,932 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:43:53,932 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:53,932 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:53,932 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:53,933 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:53,933 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:53,933 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T14:43:53,934 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T14:43:53,948 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T14:43:53,948 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60294457, jitterRate=-0.10154257714748383}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T14:43:53,950 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733496233902Initializing all the Stores at 1733496233902Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496233902Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496233903 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496233903Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496233903Cleaning up temporary data from old regions at 1733496233933 (+30 ms)Region opened successfully at 1733496233949 (+16 ms) 2024-12-06T14:43:53,960 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-06T14:43:53,964 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a6647d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bb7ea3e46d9c/172.17.0.2:0 2024-12-06T14:43:53,965 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-06T14:43:53,965 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-06T14:43:53,965 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-06T14:43:53,965 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-06T14:43:53,966 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-06T14:43:53,966 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-06T14:43:53,966 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-06T14:43:53,968 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-06T14:43:53,970 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-06T14:43:53,980 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-06T14:43:53,980 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-06T14:43:53,981 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-06T14:43:53,990 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-06T14:43:53,991 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-06T14:43:53,992 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-06T14:43:54,001 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-06T14:43:54,002 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-06T14:43:54,011 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-06T14:43:54,014 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-06T14:43:54,027 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-06T14:43:54,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T14:43:54,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41371-0x101ab19595c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T14:43:54,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T14:43:54,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41371-0x101ab19595c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:54,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:54,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:54,038 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=bb7ea3e46d9c,42883,1733496233416, sessionid=0x101ab19595c0000, setting cluster-up flag (Was=false) 2024-12-06T14:43:54,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:54,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:54,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41371-0x101ab19595c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:54,090 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-06T14:43:54,092 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bb7ea3e46d9c,42883,1733496233416 2024-12-06T14:43:54,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41371-0x101ab19595c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:54,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:54,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:54,143 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-06T14:43:54,144 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bb7ea3e46d9c,42883,1733496233416 2024-12-06T14:43:54,145 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-06T14:43:54,148 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-06T14:43:54,149 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-06T14:43:54,149 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-06T14:43:54,149 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: bb7ea3e46d9c,42883,1733496233416 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-06T14:43:54,150 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/bb7ea3e46d9c:0, corePoolSize=5, maxPoolSize=5 2024-12-06T14:43:54,151 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/bb7ea3e46d9c:0, corePoolSize=5, maxPoolSize=5 2024-12-06T14:43:54,151 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/bb7ea3e46d9c:0, corePoolSize=5, maxPoolSize=5 2024-12-06T14:43:54,151 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/bb7ea3e46d9c:0, corePoolSize=5, maxPoolSize=5 2024-12-06T14:43:54,151 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/bb7ea3e46d9c:0, corePoolSize=10, maxPoolSize=10 2024-12-06T14:43:54,151 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,151 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/bb7ea3e46d9c:0, corePoolSize=2, maxPoolSize=2 2024-12-06T14:43:54,151 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,158 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733496264157 2024-12-06T14:43:54,158 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T14:43:54,158 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-06T14:43:54,158 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-06T14:43:54,158 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-06T14:43:54,158 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-06T14:43:54,158 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-06T14:43:54,158 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-06T14:43:54,158 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-06T14:43:54,158 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:54,159 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-06T14:43:54,172 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,173 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-06T14:43:54,173 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-06T14:43:54,173 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-06T14:43:54,173 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-06T14:43:54,173 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-06T14:43:54,174 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.large.0-1733496234173,5,FailOnTimeoutGroup] 2024-12-06T14:43:54,174 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.small.0-1733496234174,5,FailOnTimeoutGroup] 2024-12-06T14:43:54,174 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,174 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-06T14:43:54,174 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,174 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,190 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-06T14:43:54,190 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f 2024-12-06T14:43:54,197 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:43:54,198 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-06T14:43:54,201 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T14:43:54,201 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:54,201 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:54,201 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-06T14:43:54,207 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T14:43:54,207 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:54,208 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:54,208 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-06T14:43:54,213 INFO [RS:0;bb7ea3e46d9c:41829 {}] regionserver.HRegionServer(746): ClusterId : 3baa005d-7704-4d42-9cf6-fd7613fbd907 2024-12-06T14:43:54,213 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T14:43:54,213 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.HRegionServer(746): ClusterId : 3baa005d-7704-4d42-9cf6-fd7613fbd907 2024-12-06T14:43:54,213 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T14:43:54,217 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T14:43:54,217 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:54,217 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:54,217 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-06T14:43:54,221 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T14:43:54,221 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:54,221 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:54,221 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T14:43:54,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740 2024-12-06T14:43:54,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740 2024-12-06T14:43:54,223 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T14:43:54,223 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T14:43:54,223 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T14:43:54,224 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T14:43:54,231 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T14:43:54,231 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62722074, jitterRate=-0.06536826491355896}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T14:43:54,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733496234197Initializing all the Stores at 1733496234197Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496234197Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496234197Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496234197Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496234197Cleaning up temporary data from old regions at 1733496234223 (+26 ms)Region opened successfully at 1733496234232 (+9 ms) 2024-12-06T14:43:54,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T14:43:54,232 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T14:43:54,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T14:43:54,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T14:43:54,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T14:43:54,233 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T14:43:54,233 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T14:43:54,233 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733496234232Disabling compacts and flushes for region at 1733496234232Disabling writes for close at 1733496234232Writing region close event to WAL at 1733496234233 (+1 ms)Closed at 1733496234233 2024-12-06T14:43:54,233 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T14:43:54,233 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T14:43:54,233 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T14:43:54,234 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T14:43:54,234 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-06T14:43:54,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-06T14:43:54,236 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T14:43:54,237 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-06T14:43:54,254 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T14:43:54,254 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T14:43:54,255 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f8455e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bb7ea3e46d9c/172.17.0.2:0 2024-12-06T14:43:54,255 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319d8c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bb7ea3e46d9c/172.17.0.2:0 2024-12-06T14:43:54,267 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;bb7ea3e46d9c:41371 2024-12-06T14:43:54,267 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T14:43:54,267 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T14:43:54,267 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T14:43:54,268 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.HRegionServer(2659): reportForDuty to master=bb7ea3e46d9c,42883,1733496233416 with port=41371, startcode=1733496233659 2024-12-06T14:43:54,269 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T14:43:54,271 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;bb7ea3e46d9c:41829 2024-12-06T14:43:54,271 INFO [RS:0;bb7ea3e46d9c:41829 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T14:43:54,271 INFO [RS:0;bb7ea3e46d9c:41829 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T14:43:54,271 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T14:43:54,272 INFO [RS:0;bb7ea3e46d9c:41829 {}] regionserver.HRegionServer(2659): reportForDuty to master=bb7ea3e46d9c,42883,1733496233416 with port=41829, startcode=1733496233612 2024-12-06T14:43:54,272 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T14:43:54,275 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37251, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T14:43:54,275 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42883 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bb7ea3e46d9c,41371,1733496233659 2024-12-06T14:43:54,275 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42883 {}] master.ServerManager(517): Registering regionserver=bb7ea3e46d9c,41371,1733496233659 2024-12-06T14:43:54,279 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56339, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T14:43:54,280 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42883 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bb7ea3e46d9c,41829,1733496233612 2024-12-06T14:43:54,280 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42883 {}] master.ServerManager(517): Registering regionserver=bb7ea3e46d9c,41829,1733496233612 2024-12-06T14:43:54,280 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f 2024-12-06T14:43:54,280 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-06T14:43:54,280 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T14:43:54,283 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f 2024-12-06T14:43:54,284 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-06T14:43:54,284 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T14:43:54,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T14:43:54,330 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] zookeeper.ZKUtil(111): regionserver:41371-0x101ab19595c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bb7ea3e46d9c,41371,1733496233659 2024-12-06T14:43:54,330 WARN [RS:1;bb7ea3e46d9c:41371 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T14:43:54,330 INFO [RS:1;bb7ea3e46d9c:41371 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T14:43:54,331 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] zookeeper.ZKUtil(111): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bb7ea3e46d9c,41829,1733496233612 2024-12-06T14:43:54,331 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bb7ea3e46d9c,41371,1733496233659] 2024-12-06T14:43:54,331 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,41371,1733496233659 2024-12-06T14:43:54,331 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bb7ea3e46d9c,41829,1733496233612] 2024-12-06T14:43:54,331 WARN [RS:0;bb7ea3e46d9c:41829 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T14:43:54,331 INFO [RS:0;bb7ea3e46d9c:41829 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T14:43:54,331 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,41829,1733496233612 2024-12-06T14:43:54,336 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T14:43:54,339 INFO [RS:0;bb7ea3e46d9c:41829 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T14:43:54,339 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T14:43:54,339 INFO [RS:1;bb7ea3e46d9c:41371 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T14:43:54,339 INFO [RS:1;bb7ea3e46d9c:41371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,339 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T14:43:54,340 INFO [RS:1;bb7ea3e46d9c:41371 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T14:43:54,340 INFO [RS:1;bb7ea3e46d9c:41371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,340 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,340 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,340 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,340 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,340 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,341 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bb7ea3e46d9c:0, corePoolSize=2, maxPoolSize=2 2024-12-06T14:43:54,341 INFO [RS:0;bb7ea3e46d9c:41829 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T14:43:54,341 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,341 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,341 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,341 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,341 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,341 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,341 INFO [RS:0;bb7ea3e46d9c:41829 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T14:43:54,341 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bb7ea3e46d9c:0, corePoolSize=3, maxPoolSize=3 2024-12-06T14:43:54,341 INFO [RS:0;bb7ea3e46d9c:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,341 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0, corePoolSize=3, maxPoolSize=3 2024-12-06T14:43:54,341 INFO [RS:0;bb7ea3e46d9c:41829 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T14:43:54,342 INFO [RS:1;bb7ea3e46d9c:41371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,342 INFO [RS:1;bb7ea3e46d9c:41371 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,342 INFO [RS:1;bb7ea3e46d9c:41371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,342 INFO [RS:1;bb7ea3e46d9c:41371 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,342 INFO [RS:1;bb7ea3e46d9c:41371 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,342 INFO [RS:1;bb7ea3e46d9c:41371 {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,41371,1733496233659-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T14:43:54,342 INFO [RS:0;bb7ea3e46d9c:41829 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T14:43:54,342 INFO [RS:0;bb7ea3e46d9c:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,342 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,342 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,342 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,342 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,343 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,343 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bb7ea3e46d9c:0, corePoolSize=2, maxPoolSize=2 2024-12-06T14:43:54,343 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,343 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,343 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,343 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,343 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,343 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:54,343 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bb7ea3e46d9c:0, corePoolSize=3, maxPoolSize=3 2024-12-06T14:43:54,343 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0, corePoolSize=3, maxPoolSize=3 2024-12-06T14:43:54,348 INFO [RS:0;bb7ea3e46d9c:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,348 INFO [RS:0;bb7ea3e46d9c:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,348 INFO [RS:0;bb7ea3e46d9c:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,348 INFO [RS:0;bb7ea3e46d9c:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,349 INFO [RS:0;bb7ea3e46d9c:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,349 INFO [RS:0;bb7ea3e46d9c:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,41829,1733496233612-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T14:43:54,363 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T14:43:54,363 INFO [RS:1;bb7ea3e46d9c:41371 {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,41371,1733496233659-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,363 INFO [RS:1;bb7ea3e46d9c:41371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,363 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.Replication(171): bb7ea3e46d9c,41371,1733496233659 started 2024-12-06T14:43:54,374 INFO [RS:0;bb7ea3e46d9c:41829 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T14:43:54,374 INFO [RS:0;bb7ea3e46d9c:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,41829,1733496233612-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,374 INFO [RS:0;bb7ea3e46d9c:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,374 INFO [RS:0;bb7ea3e46d9c:41829 {}] regionserver.Replication(171): bb7ea3e46d9c,41829,1733496233612 started 2024-12-06T14:43:54,379 INFO [RS:1;bb7ea3e46d9c:41371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,379 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.HRegionServer(1482): Serving as bb7ea3e46d9c,41371,1733496233659, RpcServer on bb7ea3e46d9c/172.17.0.2:41371, sessionid=0x101ab19595c0002 2024-12-06T14:43:54,379 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T14:43:54,379 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bb7ea3e46d9c,41371,1733496233659 2024-12-06T14:43:54,379 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bb7ea3e46d9c,41371,1733496233659' 2024-12-06T14:43:54,379 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T14:43:54,380 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T14:43:54,381 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T14:43:54,381 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T14:43:54,381 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bb7ea3e46d9c,41371,1733496233659 2024-12-06T14:43:54,381 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bb7ea3e46d9c,41371,1733496233659' 2024-12-06T14:43:54,381 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T14:43:54,382 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T14:43:54,382 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T14:43:54,382 INFO [RS:1;bb7ea3e46d9c:41371 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T14:43:54,382 INFO [RS:1;bb7ea3e46d9c:41371 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T14:43:54,388 WARN [bb7ea3e46d9c:42883 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T14:43:54,395 INFO [RS:0;bb7ea3e46d9c:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,395 INFO [RS:0;bb7ea3e46d9c:41829 {}] regionserver.HRegionServer(1482): Serving as bb7ea3e46d9c,41829,1733496233612, RpcServer on bb7ea3e46d9c/172.17.0.2:41829, sessionid=0x101ab19595c0001 2024-12-06T14:43:54,395 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T14:43:54,395 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bb7ea3e46d9c,41829,1733496233612 2024-12-06T14:43:54,395 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bb7ea3e46d9c,41829,1733496233612' 2024-12-06T14:43:54,396 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T14:43:54,398 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T14:43:54,400 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T14:43:54,400 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T14:43:54,400 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bb7ea3e46d9c,41829,1733496233612 2024-12-06T14:43:54,400 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bb7ea3e46d9c,41829,1733496233612' 2024-12-06T14:43:54,400 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T14:43:54,401 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T14:43:54,401 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T14:43:54,401 INFO [RS:0;bb7ea3e46d9c:41829 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T14:43:54,401 INFO [RS:0;bb7ea3e46d9c:41829 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T14:43:54,483 INFO [RS:1;bb7ea3e46d9c:41371 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T14:43:54,484 INFO [RS:1;bb7ea3e46d9c:41371 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bb7ea3e46d9c%2C41371%2C1733496233659, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,41371,1733496233659, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/oldWALs, maxLogs=55 2024-12-06T14:43:54,502 INFO [RS:0;bb7ea3e46d9c:41829 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T14:43:54,503 INFO [RS:0;bb7ea3e46d9c:41829 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bb7ea3e46d9c%2C41829%2C1733496233612, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,41829,1733496233612, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/oldWALs, maxLogs=55 2024-12-06T14:43:54,516 INFO [RS:1;bb7ea3e46d9c:41371 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,41371,1733496233659/bb7ea3e46d9c%2C41371%2C1733496233659.1733496234486 2024-12-06T14:43:54,516 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T14:43:54,523 INFO [RS:0;bb7ea3e46d9c:41829 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,41829,1733496233612/bb7ea3e46d9c%2C41829%2C1733496233612.1733496234504 2024-12-06T14:43:54,528 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T14:43:54,638 DEBUG [bb7ea3e46d9c:42883 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-06T14:43:54,638 DEBUG [bb7ea3e46d9c:42883 {}] balancer.BalancerClusterState(204): Hosts are {bb7ea3e46d9c=0} racks are {/default-rack=0} 2024-12-06T14:43:54,639 DEBUG [bb7ea3e46d9c:42883 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T14:43:54,639 DEBUG [bb7ea3e46d9c:42883 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T14:43:54,639 DEBUG [bb7ea3e46d9c:42883 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T14:43:54,639 DEBUG [bb7ea3e46d9c:42883 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T14:43:54,640 INFO [bb7ea3e46d9c:42883 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T14:43:54,640 INFO [bb7ea3e46d9c:42883 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T14:43:54,640 DEBUG [bb7ea3e46d9c:42883 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T14:43:54,640 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=bb7ea3e46d9c,41371,1733496233659 2024-12-06T14:43:54,641 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bb7ea3e46d9c,41371,1733496233659, state=OPENING 2024-12-06T14:43:54,658 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-06T14:43:54,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:54,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:54,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41371-0x101ab19595c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:54,670 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:43:54,671 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:43:54,671 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:43:54,672 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T14:43:54,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=bb7ea3e46d9c,41371,1733496233659}] 2024-12-06T14:43:54,826 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T14:43:54,827 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43561, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T14:43:54,830 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-06T14:43:54,830 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T14:43:54,830 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-06T14:43:54,831 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bb7ea3e46d9c%2C41371%2C1733496233659.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,41371,1733496233659, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/oldWALs, maxLogs=55 2024-12-06T14:43:54,854 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,41371,1733496233659/bb7ea3e46d9c%2C41371%2C1733496233659.meta.1733496234832.meta 2024-12-06T14:43:54,854 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T14:43:54,854 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-06T14:43:54,855 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-06T14:43:54,855 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-06T14:43:54,855 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-06T14:43:54,855 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-06T14:43:54,855 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:43:54,855 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-06T14:43:54,855 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-06T14:43:54,857 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-06T14:43:54,858 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T14:43:54,858 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:54,858 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:54,858 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-06T14:43:54,859 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T14:43:54,859 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:54,859 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:54,859 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-06T14:43:54,860 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T14:43:54,860 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:54,861 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:54,861 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-06T14:43:54,861 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T14:43:54,861 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:54,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:54,862 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T14:43:54,863 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740 2024-12-06T14:43:54,863 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740 2024-12-06T14:43:54,864 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T14:43:54,864 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T14:43:54,864 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T14:43:54,865 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T14:43:54,865 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74648804, jitterRate=0.1123538613319397}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T14:43:54,865 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-06T14:43:54,866 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733496234856Writing region info on filesystem at 1733496234856Initializing all the Stores at 1733496234856Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496234856Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496234856Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496234856Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496234856Cleaning up temporary data from old regions at 1733496234864 (+8 ms)Running coprocessor post-open hooks at 1733496234865 (+1 ms)Region opened successfully at 1733496234866 (+1 ms) 2024-12-06T14:43:54,867 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733496234825 2024-12-06T14:43:54,870 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,41371,1733496233659 2024-12-06T14:43:54,871 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-06T14:43:54,871 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-06T14:43:54,871 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bb7ea3e46d9c,41371,1733496233659, state=OPEN 2024-12-06T14:43:54,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T14:43:54,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T14:43:54,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41371-0x101ab19595c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T14:43:54,880 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=bb7ea3e46d9c,41371,1733496233659 2024-12-06T14:43:54,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:43:54,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:43:54,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:43:54,882 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-06T14:43:54,882 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bb7ea3e46d9c,41371,1733496233659 in 208 msec 2024-12-06T14:43:54,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-06T14:43:54,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 648 msec 2024-12-06T14:43:54,884 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T14:43:54,884 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-06T14:43:54,887 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:54,887 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,41371,1733496233659, seqNum=-1] 2024-12-06T14:43:54,888 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:54,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52755, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:54,897 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 750 msec 2024-12-06T14:43:54,897 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733496234897, completionTime=-1 2024-12-06T14:43:54,897 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-06T14:43:54,898 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-06T14:43:54,900 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-06T14:43:54,901 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733496294901 2024-12-06T14:43:54,901 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733496354901 2024-12-06T14:43:54,901 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-12-06T14:43:54,902 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,42883,1733496233416-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,902 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,42883,1733496233416-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,902 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,42883,1733496233416-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,902 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-bb7ea3e46d9c:42883, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,902 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,904 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,906 DEBUG [master/bb7ea3e46d9c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-06T14:43:54,910 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.155sec 2024-12-06T14:43:54,910 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-06T14:43:54,910 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-06T14:43:54,910 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-06T14:43:54,910 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-06T14:43:54,910 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-06T14:43:54,910 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,42883,1733496233416-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T14:43:54,910 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,42883,1733496233416-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-06T14:43:54,911 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-06T14:43:54,911 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-06T14:43:54,911 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,42883,1733496233416-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:54,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc06048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:54,916 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,42883,-1 for getting cluster id 2024-12-06T14:43:54,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T14:43:54,917 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3baa005d-7704-4d42-9cf6-fd7613fbd907' 2024-12-06T14:43:54,918 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T14:43:54,918 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3baa005d-7704-4d42-9cf6-fd7613fbd907" 2024-12-06T14:43:54,919 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@358c0e00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:54,919 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,42883,-1] 2024-12-06T14:43:54,919 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T14:43:54,920 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:54,922 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T14:43:54,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143f7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T14:43:54,924 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:54,925 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,41371,1733496233659, seqNum=-1] 2024-12-06T14:43:54,926 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:54,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:54,931 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=bb7ea3e46d9c,42883,1733496233416 2024-12-06T14:43:54,932 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T14:43:54,933 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is bb7ea3e46d9c,42883,1733496233416 2024-12-06T14:43:54,934 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1f88392f 2024-12-06T14:43:54,934 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T14:43:54,936 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T14:43:54,937 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42883 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733496234931', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T14:43:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42883 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733496234931 2024-12-06T14:43:54,940 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733496234931 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T14:43:54,940 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:54,941 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733496234931 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T14:43:54,941 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42883 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733496234931" procId is: 4 2024-12-06T14:43:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T14:43:54,955 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733496234931-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e3343eea25c5bc5cb9c4ce62f90e1fd6, NAME => 'testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733496234931', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f 2024-12-06T14:43:54,964 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733496234931-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:43:54,964 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733496234931-pool-0 {}] regionserver.HRegion(1722): Closing e3343eea25c5bc5cb9c4ce62f90e1fd6, disabling compactions & flushes 2024-12-06T14:43:54,964 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733496234931-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6. 2024-12-06T14:43:54,964 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733496234931-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6. 2024-12-06T14:43:54,965 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733496234931-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6. after waiting 0 ms 2024-12-06T14:43:54,965 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733496234931-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6. 2024-12-06T14:43:54,965 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733496234931-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6. 2024-12-06T14:43:54,965 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733496234931-pool-0 {}] regionserver.HRegion(1676): Region close journal for e3343eea25c5bc5cb9c4ce62f90e1fd6: Waiting for close lock at 1733496234964Disabling compacts and flushes for region at 1733496234964Disabling writes for close at 1733496234965 (+1 ms)Writing region close event to WAL at 1733496234965Closed at 1733496234965 2024-12-06T14:43:54,966 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733496234931 execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T14:43:54,967 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733496234966"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733496234966"}]},"ts":"1733496234966"} 2024-12-06T14:43:54,970 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T14:43:54,971 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733496234931 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T14:43:54,972 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733496234931","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496234971"}]},"ts":"1733496234971"} 2024-12-06T14:43:54,974 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733496234931, state=ENABLING in hbase:meta 2024-12-06T14:43:54,975 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {bb7ea3e46d9c=0} racks are {/default-rack=0} 2024-12-06T14:43:54,975 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T14:43:54,975 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T14:43:54,975 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T14:43:54,975 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T14:43:54,975 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T14:43:54,976 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T14:43:54,976 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T14:43:54,976 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733496234931, region=e3343eea25c5bc5cb9c4ce62f90e1fd6, ASSIGN}] 2024-12-06T14:43:54,977 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733496234931, region=e3343eea25c5bc5cb9c4ce62f90e1fd6, ASSIGN 2024-12-06T14:43:54,978 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733496234931, region=e3343eea25c5bc5cb9c4ce62f90e1fd6, ASSIGN; state=OFFLINE, location=bb7ea3e46d9c,41371,1733496233659; forceNewPlan=false, retain=false 2024-12-06T14:43:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T14:43:55,129 INFO [bb7ea3e46d9c:42883 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-06T14:43:55,129 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e3343eea25c5bc5cb9c4ce62f90e1fd6, regionState=OPENING, regionLocation=bb7ea3e46d9c,41371,1733496233659 2024-12-06T14:43:55,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733496234931, region=e3343eea25c5bc5cb9c4ce62f90e1fd6, ASSIGN because future has completed 2024-12-06T14:43:55,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e3343eea25c5bc5cb9c4ce62f90e1fd6, server=bb7ea3e46d9c,41371,1733496233659}] 2024-12-06T14:43:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T14:43:55,287 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6. 2024-12-06T14:43:55,288 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e3343eea25c5bc5cb9c4ce62f90e1fd6, NAME => 'testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6.', STARTKEY => '', ENDKEY => ''} 2024-12-06T14:43:55,288 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733496234931 e3343eea25c5bc5cb9c4ce62f90e1fd6 2024-12-06T14:43:55,288 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:43:55,288 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e3343eea25c5bc5cb9c4ce62f90e1fd6 2024-12-06T14:43:55,288 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e3343eea25c5bc5cb9c4ce62f90e1fd6 2024-12-06T14:43:55,289 INFO [StoreOpener-e3343eea25c5bc5cb9c4ce62f90e1fd6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region e3343eea25c5bc5cb9c4ce62f90e1fd6 2024-12-06T14:43:55,294 INFO [StoreOpener-e3343eea25c5bc5cb9c4ce62f90e1fd6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3343eea25c5bc5cb9c4ce62f90e1fd6 columnFamilyName family 2024-12-06T14:43:55,294 DEBUG [StoreOpener-e3343eea25c5bc5cb9c4ce62f90e1fd6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:55,294 INFO [StoreOpener-e3343eea25c5bc5cb9c4ce62f90e1fd6-1 {}] regionserver.HStore(327): Store=e3343eea25c5bc5cb9c4ce62f90e1fd6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T14:43:55,294 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e3343eea25c5bc5cb9c4ce62f90e1fd6 2024-12-06T14:43:55,295 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnGracefulStop1733496234931/e3343eea25c5bc5cb9c4ce62f90e1fd6 2024-12-06T14:43:55,295 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnGracefulStop1733496234931/e3343eea25c5bc5cb9c4ce62f90e1fd6 2024-12-06T14:43:55,295 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e3343eea25c5bc5cb9c4ce62f90e1fd6 2024-12-06T14:43:55,295 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e3343eea25c5bc5cb9c4ce62f90e1fd6 2024-12-06T14:43:55,296 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e3343eea25c5bc5cb9c4ce62f90e1fd6 2024-12-06T14:43:55,303 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnGracefulStop1733496234931/e3343eea25c5bc5cb9c4ce62f90e1fd6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T14:43:55,303 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e3343eea25c5bc5cb9c4ce62f90e1fd6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69214560, jitterRate=0.031377315521240234}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T14:43:55,303 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e3343eea25c5bc5cb9c4ce62f90e1fd6 2024-12-06T14:43:55,304 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e3343eea25c5bc5cb9c4ce62f90e1fd6: Running coprocessor pre-open hook at 1733496235288Writing region info on filesystem at 1733496235288Initializing all the Stores at 1733496235289 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496235289Cleaning up temporary data from old regions at 1733496235295 (+6 ms)Running coprocessor post-open hooks at 1733496235303 (+8 ms)Region opened successfully at 1733496235304 (+1 ms) 2024-12-06T14:43:55,305 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6., pid=6, masterSystemTime=1733496235285 2024-12-06T14:43:55,309 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6. 2024-12-06T14:43:55,309 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6. 2024-12-06T14:43:55,309 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e3343eea25c5bc5cb9c4ce62f90e1fd6, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,41371,1733496233659 2024-12-06T14:43:55,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e3343eea25c5bc5cb9c4ce62f90e1fd6, server=bb7ea3e46d9c,41371,1733496233659 because future has completed 2024-12-06T14:43:55,316 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-06T14:43:55,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e3343eea25c5bc5cb9c4ce62f90e1fd6, server=bb7ea3e46d9c,41371,1733496233659 in 181 msec 2024-12-06T14:43:55,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-06T14:43:55,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733496234931, region=e3343eea25c5bc5cb9c4ce62f90e1fd6, ASSIGN in 341 msec 2024-12-06T14:43:55,321 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733496234931 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T14:43:55,321 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733496234931","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733496235321"}]},"ts":"1733496235321"} 2024-12-06T14:43:55,325 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733496234931, state=ENABLED in hbase:meta 2024-12-06T14:43:55,327 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733496234931 execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T14:43:55,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733496234931 in 390 msec 2024-12-06T14:43:55,416 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T14:43:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T14:43:55,575 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733496234931 completed 2024-12-06T14:43:55,575 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733496234931 get assigned. Timeout = 60000ms 2024-12-06T14:43:55,575 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T14:43:55,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733496234931 assigned to meta. Checking AM states. 2024-12-06T14:43:55,580 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T14:43:55,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733496234931 assigned. 2024-12-06T14:43:55,584 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733496234931', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6., hostname=bb7ea3e46d9c,41371,1733496233659, seqNum=2] 2024-12-06T14:43:55,589 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e3343eea25c5bc5cb9c4ce62f90e1fd6 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-06T14:43:55,599 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on dcc17cc208134c2aa5a26aabef2238e0 2024-12-06T14:43:55,617 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnGracefulStop1733496234931/e3343eea25c5bc5cb9c4ce62f90e1fd6/.tmp/family/dcc17cc208134c2aa5a26aabef2238e0 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-06T14:43:55,617 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnGracefulStop1733496234931/e3343eea25c5bc5cb9c4ce62f90e1fd6/.tmp/family/dcc17cc208134c2aa5a26aabef2238e0 2024-12-06T14:43:55,621 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnGracefulStop1733496234931/e3343eea25c5bc5cb9c4ce62f90e1fd6/.tmp/family/dcc17cc208134c2aa5a26aabef2238e0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnGracefulStop1733496234931/e3343eea25c5bc5cb9c4ce62f90e1fd6/family/dcc17cc208134c2aa5a26aabef2238e0 2024-12-06T14:43:55,623 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnGracefulStop1733496234931/e3343eea25c5bc5cb9c4ce62f90e1fd6/family/dcc17cc208134c2aa5a26aabef2238e0, delay=992 ms 2024-12-06T14:43:55,624 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnGracefulStop1733496234931/e3343eea25c5bc5cb9c4ce62f90e1fd6/family/dcc17cc208134c2aa5a26aabef2238e0, entries=2, sequenceid=6, filesize=4.9 K 2024-12-06T14:43:55,625 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for e3343eea25c5bc5cb9c4ce62f90e1fd6 in 36ms, sequenceid=6, compaction requested=false 2024-12-06T14:43:55,625 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733496234931' 2024-12-06T14:43:55,626 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e3343eea25c5bc5cb9c4ce62f90e1fd6: 2024-12-06T14:43:56,616 DEBUG [hfile-prefetch-1733496231156 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key dcc17cc208134c2aa5a26aabef2238e0_0. 2024-12-06T14:43:56,616 DEBUG [hfile-prefetch-1733496231156 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key dcc17cc208134c2aa5a26aabef2238e0_117. 2024-12-06T14:43:56,616 DEBUG [hfile-prefetch-1733496231156 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnGracefulStop1733496234931/e3343eea25c5bc5cb9c4ce62f90e1fd6/family/dcc17cc208134c2aa5a26aabef2238e0, with total blocks 2, and data blocks 1 2024-12-06T14:43:56,616 DEBUG [hfile-prefetch-1733496231156 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-06T14:43:56,616 DEBUG [hfile-prefetch-1733496231156 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnGracefulStop1733496234931/e3343eea25c5bc5cb9c4ce62f90e1fd6/family/dcc17cc208134c2aa5a26aabef2238e0, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-06T14:43:56,616 DEBUG [hfile-prefetch-1733496231156 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnGracefulStop1733496234931/e3343eea25c5bc5cb9c4ce62f90e1fd6/family/dcc17cc208134c2aa5a26aabef2238e0. Will sleep 100ms and try the verification again. 2024-12-06T14:43:56,626 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;bb7ea3e46d9c:41371,5,FailOnTimeoutGroup] 2024-12-06T14:43:56,627 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bb7ea3e46d9c,41371,1733496233659' ***** 2024-12-06T14:43:56,627 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-12-06T14:43:56,627 INFO [RS:1;bb7ea3e46d9c:41371 {}] bucket.BucketCache(1738): Disabling cache 2024-12-06T14:43:56,644 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T14:43:56,647 INFO [regionserver/bb7ea3e46d9c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T14:43:56,710 INFO [RS:1;bb7ea3e46d9c:41371 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30/bucket.persistence 2024-12-06T14:43:56,710 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-06T14:43:56,710 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-06T14:43:56,710 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-06T14:43:56,710 INFO [RS:1;bb7ea3e46d9c:41371 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-06T14:43:56,710 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-06T14:43:56,717 DEBUG [hfile-prefetch-1733496231156 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnGracefulStop1733496234931/e3343eea25c5bc5cb9c4ce62f90e1fd6/family/dcc17cc208134c2aa5a26aabef2238e0, with total blocks 2, and data blocks 1 2024-12-06T14:43:56,717 DEBUG [hfile-prefetch-1733496231156 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-12-06T14:43:56,717 DEBUG [hfile-prefetch-1733496231156 {}] bucket.BucketCache(2293): found block for file dcc17cc208134c2aa5a26aabef2238e0 in the backing map. Acquiring read lock for offset 0 2024-12-06T14:43:56,717 DEBUG [hfile-prefetch-1733496231156 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnGracefulStop1733496234931/e3343eea25c5bc5cb9c4ce62f90e1fd6/family/dcc17cc208134c2aa5a26aabef2238e0 has now been fully cached. 2024-12-06T14:43:56,717 DEBUG [hfile-prefetch-1733496231156 {}] hfile.PrefetchExecutor(121): Prefetch completed for dcc17cc208134c2aa5a26aabef2238e0 2024-12-06T14:43:56,721 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] bucket.BucketCache(1487): Thread RS:1;bb7ea3e46d9c:41371 started persisting bucket cache to file 2024-12-06T14:43:56,721 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T14:43:56,721 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-12-06T14:43:56,724 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-12-06T14:43:56,724 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1733496236721 2024-12-06T14:43:56,724 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] bucket.BucketCache(1509): Thread RS:1;bb7ea3e46d9c:41371 finished persisting bucket cache to file, renaming 2024-12-06T14:43:56,724 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T14:43:56,724 INFO [RS:1;bb7ea3e46d9c:41371 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T14:43:56,724 INFO [RS:1;bb7ea3e46d9c:41371 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T14:43:56,724 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.HRegionServer(3091): Received CLOSE for e3343eea25c5bc5cb9c4ce62f90e1fd6 2024-12-06T14:43:56,725 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.HRegionServer(959): stopping server bb7ea3e46d9c,41371,1733496233659 2024-12-06T14:43:56,725 INFO [RS:1;bb7ea3e46d9c:41371 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T14:43:56,725 INFO [RS:1;bb7ea3e46d9c:41371 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;bb7ea3e46d9c:41371. 2024-12-06T14:43:56,725 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-06T14:43:56,725 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e3343eea25c5bc5cb9c4ce62f90e1fd6, disabling compactions & flushes 2024-12-06T14:43:56,725 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T14:43:56,725 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6. 2024-12-06T14:43:56,725 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:56,725 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6. 2024-12-06T14:43:56,725 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6. after waiting 0 ms 2024-12-06T14:43:56,725 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T14:43:56,725 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T14:43:56,725 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6. 2024-12-06T14:43:56,725 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T14:43:56,725 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-06T14:43:56,726 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-06T14:43:56,727 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, e3343eea25c5bc5cb9c4ce62f90e1fd6=testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6.} 2024-12-06T14:43:56,727 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-06T14:43:56,727 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, e3343eea25c5bc5cb9c4ce62f90e1fd6 2024-12-06T14:43:56,727 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T14:43:56,727 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T14:43:56,727 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T14:43:56,727 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T14:43:56,727 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T14:43:56,727 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-12-06T14:43:56,733 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-06T14:43:56,733 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-06T14:43:56,733 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T14:43:56,733 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T14:43:56,736 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-06T14:43:56,736 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733496236733 2024-12-06T14:43:56,737 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-06T14:43:56,737 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/default/testBlockEvictionOnGracefulStop1733496234931/e3343eea25c5bc5cb9c4ce62f90e1fd6/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-06T14:43:56,738 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6. 2024-12-06T14:43:56,738 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e3343eea25c5bc5cb9c4ce62f90e1fd6: Waiting for close lock at 1733496236725Running coprocessor pre-close hooks at 1733496236725Disabling compacts and flushes for region at 1733496236725Disabling writes for close at 1733496236725Writing region close event to WAL at 1733496236728 (+3 ms)Running coprocessor post-close hooks at 1733496236738 (+10 ms)Closed at 1733496236738 2024-12-06T14:43:56,738 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6. 2024-12-06T14:43:56,747 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c4da72a4eeae4da18a09d3cb081c0d15 2024-12-06T14:43:56,765 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/info/c4da72a4eeae4da18a09d3cb081c0d15 is 209, key is testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6./info:regioninfo/1733496235309/Put/seqid=0 2024-12-06T14:43:56,766 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/info/c4da72a4eeae4da18a09d3cb081c0d15 2024-12-06T14:43:56,774 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9165a473c2914db989931fcff2ea5665 2024-12-06T14:43:56,792 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/ns/9165a473c2914db989931fcff2ea5665 is 43, key is default/ns:d/1733496234891/Put/seqid=0 2024-12-06T14:43:56,793 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/ns/9165a473c2914db989931fcff2ea5665 2024-12-06T14:43:56,802 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0aa4aebf6f524063a4788826a2aa768f 2024-12-06T14:43:56,817 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/table/0aa4aebf6f524063a4788826a2aa768f is 80, key is testBlockEvictionOnGracefulStop1733496234931/table:state/1733496235321/Put/seqid=0 2024-12-06T14:43:56,818 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/table/0aa4aebf6f524063a4788826a2aa768f 2024-12-06T14:43:56,826 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/info/c4da72a4eeae4da18a09d3cb081c0d15 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/info/c4da72a4eeae4da18a09d3cb081c0d15 2024-12-06T14:43:56,828 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/info/c4da72a4eeae4da18a09d3cb081c0d15, delay=988 ms 2024-12-06T14:43:56,829 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/info/c4da72a4eeae4da18a09d3cb081c0d15, entries=10, sequenceid=11, filesize=7.0 K 2024-12-06T14:43:56,830 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/ns/9165a473c2914db989931fcff2ea5665 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/ns/9165a473c2914db989931fcff2ea5665 2024-12-06T14:43:56,831 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/ns/9165a473c2914db989931fcff2ea5665, delay=913 ms 2024-12-06T14:43:56,833 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/ns/9165a473c2914db989931fcff2ea5665, entries=2, sequenceid=11, filesize=5.0 K 2024-12-06T14:43:56,833 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/table/0aa4aebf6f524063a4788826a2aa768f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/table/0aa4aebf6f524063a4788826a2aa768f 2024-12-06T14:43:56,835 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/table/0aa4aebf6f524063a4788826a2aa768f, delay=986 ms 2024-12-06T14:43:56,836 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/table/0aa4aebf6f524063a4788826a2aa768f, entries=2, sequenceid=11, filesize=5.3 K 2024-12-06T14:43:56,837 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 110ms, sequenceid=11, compaction requested=false 2024-12-06T14:43:56,838 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/info/c4da72a4eeae4da18a09d3cb081c0d15 2024-12-06T14:43:56,839 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/ns/9165a473c2914db989931fcff2ea5665 2024-12-06T14:43:56,841 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/table/0aa4aebf6f524063a4788826a2aa768f 2024-12-06T14:43:56,846 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-06T14:43:56,846 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T14:43:56,846 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T14:43:56,846 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733496236727Running coprocessor pre-close hooks at 1733496236727Disabling compacts and flushes for region at 1733496236727Disabling writes for close at 1733496236727Obtaining lock to block concurrent updates at 1733496236727Preparing flush snapshotting stores in 1588230740 at 1733496236727Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1733496236728 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733496236728Flushing 1588230740/info: creating writer at 1733496236728Flushing 1588230740/info: appending metadata at 1733496236764 (+36 ms)Flushing 1588230740/info: closing flushed file at 1733496236764Flushing 1588230740/ns: creating writer at 1733496236768 (+4 ms)Flushing 1588230740/ns: appending metadata at 1733496236792 (+24 ms)Flushing 1588230740/ns: closing flushed file at 1733496236792Flushing 1588230740/table: creating writer at 1733496236796 (+4 ms)Flushing 1588230740/table: appending metadata at 1733496236817 (+21 ms)Flushing 1588230740/table: closing flushed file at 1733496236817Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f7447d2: reopening flushed file at 1733496236826 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@629e0e33: reopening flushed file at 1733496236830 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8b50352: reopening flushed file at 1733496236833 (+3 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 110ms, sequenceid=11, compaction requested=false at 1733496236837 (+4 ms)Writing region close event to WAL at 1733496236841 (+4 ms)Running coprocessor post-close hooks at 1733496236846 (+5 ms)Closed at 1733496236846 2024-12-06T14:43:56,847 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-06T14:43:56,927 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.HRegionServer(976): stopping server bb7ea3e46d9c,41371,1733496233659; all regions closed. 2024-12-06T14:43:56,929 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/oldWALs 2024-12-06T14:43:56,929 INFO [RS:1;bb7ea3e46d9c:41371 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bb7ea3e46d9c%2C41371%2C1733496233659.meta:.meta(num 1733496234832) 2024-12-06T14:43:56,930 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/oldWALs 2024-12-06T14:43:56,931 INFO [RS:1;bb7ea3e46d9c:41371 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bb7ea3e46d9c%2C41371%2C1733496233659:(num 1733496234486) 2024-12-06T14:43:56,931 DEBUG [RS:1;bb7ea3e46d9c:41371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:56,931 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T14:43:56,931 INFO [RS:1;bb7ea3e46d9c:41371 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T14:43:56,931 INFO [RS:1;bb7ea3e46d9c:41371 {}] hbase.ChoreService(370): Chore service for: regionserver/bb7ea3e46d9c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-06T14:43:56,931 INFO [RS:1;bb7ea3e46d9c:41371 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T14:43:56,931 INFO [regionserver/bb7ea3e46d9c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T14:43:56,931 INFO [RS:1;bb7ea3e46d9c:41371 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41371 2024-12-06T14:43:56,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T14:43:56,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41371-0x101ab19595c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bb7ea3e46d9c,41371,1733496233659 2024-12-06T14:43:56,954 INFO [RS:1;bb7ea3e46d9c:41371 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T14:43:56,954 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bb7ea3e46d9c,41371,1733496233659] 2024-12-06T14:43:56,974 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bb7ea3e46d9c,41371,1733496233659 already deleted, retry=false 2024-12-06T14:43:56,974 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of bb7ea3e46d9c,41371,1733496233659 on bb7ea3e46d9c,42883,1733496233416 2024-12-06T14:43:56,979 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure bb7ea3e46d9c,41371,1733496233659, splitWal=true, meta=true 2024-12-06T14:43:56,982 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure bb7ea3e46d9c,41371,1733496233659, splitWal=true, meta=true 2024-12-06T14:43:56,982 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for bb7ea3e46d9c,41371,1733496233659 (carryingMeta=true) bb7ea3e46d9c,41371,1733496233659/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@5f01119f[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-06T14:43:56,983 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure bb7ea3e46d9c,41371,1733496233659, splitWal=true, meta=true, isMeta: true 2024-12-06T14:43:56,983 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,41371,1733496233659-splitting 2024-12-06T14:43:56,984 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,41371,1733496233659-splitting dir is empty, no logs to split. 2024-12-06T14:43:56,984 INFO [PEWorker-2 {}] master.SplitWALManager(105): bb7ea3e46d9c,41371,1733496233659 WAL count=0, meta=true 2024-12-06T14:43:56,986 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,41371,1733496233659-splitting dir is empty, no logs to split. 2024-12-06T14:43:56,986 INFO [PEWorker-2 {}] master.SplitWALManager(105): bb7ea3e46d9c,41371,1733496233659 WAL count=0, meta=true 2024-12-06T14:43:56,986 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if bb7ea3e46d9c,41371,1733496233659 WAL splitting is done? wals=0, meta=true 2024-12-06T14:43:56,988 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-06T14:43:56,989 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T14:43:56,990 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-06T14:43:57,064 INFO [RS:1;bb7ea3e46d9c:41371 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T14:43:57,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41371-0x101ab19595c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T14:43:57,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41371-0x101ab19595c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T14:43:57,064 INFO [RS:1;bb7ea3e46d9c:41371 {}] regionserver.HRegionServer(1031): Exiting; stopping=bb7ea3e46d9c,41371,1733496233659; zookeeper connection closed. 2024-12-06T14:43:57,064 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@39b120c6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@39b120c6 2024-12-06T14:43:57,140 DEBUG [bb7ea3e46d9c:42883 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-06T14:43:57,141 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=bb7ea3e46d9c,41829,1733496233612 2024-12-06T14:43:57,141 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bb7ea3e46d9c,41829,1733496233612, state=OPENING 2024-12-06T14:43:57,148 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bb7ea3e46d9c:0 server-side Connection retries=45 2024-12-06T14:43:57,148 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:57,148 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:57,148 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T14:43:57,148 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T14:43:57,148 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T14:43:57,148 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T14:43:57,149 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T14:43:57,149 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39099 2024-12-06T14:43:57,151 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39099 connecting to ZooKeeper ensemble=127.0.0.1:59695 2024-12-06T14:43:57,151 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:57,152 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:57,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T14:43:57,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T14:43:57,169 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T14:43:57,169 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:43:57,169 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:43:57,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=bb7ea3e46d9c,41829,1733496233612}] 2024-12-06T14:43:57,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:390990x0, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T14:43:57,181 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:390990x0, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-06T14:43:57,182 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39099-0x101ab19595c0003 connected 2024-12-06T14:43:57,182 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-06T14:43:57,183 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T14:43:57,183 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30/bucket.cache 2024-12-06T14:43:57,183 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-06T14:43:57,184 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-06T14:43:57,184 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-06T14:43:57,185 INFO [Thread-206 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-06T14:43:57,185 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-06T14:43:57,186 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T14:43:57,186 INFO [Thread-206 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-06T14:43:57,186 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39099-0x101ab19595c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T14:43:57,187 INFO [Thread-206 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-12-06T14:43:57,188 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39099-0x101ab19595c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T14:43:57,192 DEBUG [Thread-206 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-06T14:43:57,193 INFO [Thread-206 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-06T14:43:57,193 INFO [Thread-206 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-12-06T14:43:57,193 INFO [Thread-206 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30/bucket.persistence is complete. 2024-12-06T14:43:57,196 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39099 2024-12-06T14:43:57,204 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39099 2024-12-06T14:43:57,205 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39099 2024-12-06T14:43:57,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39099 2024-12-06T14:43:57,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39099 2024-12-06T14:43:57,213 INFO [RS:2;bb7ea3e46d9c:39099 {}] regionserver.HRegionServer(746): ClusterId : 3baa005d-7704-4d42-9cf6-fd7613fbd907 2024-12-06T14:43:57,213 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T14:43:57,224 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T14:43:57,224 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T14:43:57,239 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T14:43:57,239 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66aba377, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bb7ea3e46d9c/172.17.0.2:0 2024-12-06T14:43:57,257 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;bb7ea3e46d9c:39099 2024-12-06T14:43:57,257 INFO [RS:2;bb7ea3e46d9c:39099 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T14:43:57,257 INFO [RS:2;bb7ea3e46d9c:39099 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T14:43:57,257 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T14:43:57,258 INFO [RS:2;bb7ea3e46d9c:39099 {}] regionserver.HRegionServer(2659): reportForDuty to master=bb7ea3e46d9c,42883,1733496233416 with port=39099, startcode=1733496237147 2024-12-06T14:43:57,259 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T14:43:57,267 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33295, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T14:43:57,268 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42883 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bb7ea3e46d9c,39099,1733496237147 2024-12-06T14:43:57,268 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42883 {}] master.ServerManager(517): Registering regionserver=bb7ea3e46d9c,39099,1733496237147 2024-12-06T14:43:57,272 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f 2024-12-06T14:43:57,272 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-06T14:43:57,272 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T14:43:57,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T14:43:57,286 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bb7ea3e46d9c,39099,1733496237147] 2024-12-06T14:43:57,289 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] zookeeper.ZKUtil(111): regionserver:39099-0x101ab19595c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bb7ea3e46d9c,39099,1733496237147 2024-12-06T14:43:57,289 WARN [RS:2;bb7ea3e46d9c:39099 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T14:43:57,289 INFO [RS:2;bb7ea3e46d9c:39099 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T14:43:57,290 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,39099,1733496237147 2024-12-06T14:43:57,311 INFO [RS:2;bb7ea3e46d9c:39099 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T14:43:57,320 INFO [RS:2;bb7ea3e46d9c:39099 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T14:43:57,321 INFO [RS:2;bb7ea3e46d9c:39099 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T14:43:57,321 INFO [RS:2;bb7ea3e46d9c:39099 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:57,321 INFO [RS:2;bb7ea3e46d9c:39099 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T14:43:57,322 INFO [RS:2;bb7ea3e46d9c:39099 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T14:43:57,322 INFO [RS:2;bb7ea3e46d9c:39099 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:57,322 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:57,322 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:57,322 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:57,322 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:57,322 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:57,322 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bb7ea3e46d9c:0, corePoolSize=2, maxPoolSize=2 2024-12-06T14:43:57,322 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:57,322 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:57,323 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:57,323 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:57,323 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:57,323 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T14:43:57,323 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bb7ea3e46d9c:0, corePoolSize=3, maxPoolSize=3 2024-12-06T14:43:57,323 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0, corePoolSize=3, maxPoolSize=3 2024-12-06T14:43:57,323 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T14:43:57,327 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-06T14:43:57,327 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-06T14:43:57,332 INFO [RS:2;bb7ea3e46d9c:39099 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:57,332 INFO [RS:2;bb7ea3e46d9c:39099 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:57,332 INFO [RS:2;bb7ea3e46d9c:39099 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:57,332 INFO [RS:2;bb7ea3e46d9c:39099 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:57,332 INFO [RS:2;bb7ea3e46d9c:39099 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:57,333 INFO [RS:2;bb7ea3e46d9c:39099 {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,39099,1733496237147-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T14:43:57,350 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47723, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T14:43:57,375 INFO [RS:2;bb7ea3e46d9c:39099 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T14:43:57,376 INFO [RS:2;bb7ea3e46d9c:39099 {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,39099,1733496237147-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:57,376 INFO [RS:2;bb7ea3e46d9c:39099 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:57,376 INFO [RS:2;bb7ea3e46d9c:39099 {}] regionserver.Replication(171): bb7ea3e46d9c,39099,1733496237147 started 2024-12-06T14:43:57,384 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-06T14:43:57,384 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T14:43:57,385 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-06T14:43:57,386 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bb7ea3e46d9c%2C41829%2C1733496233612.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,41829,1733496233612, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/oldWALs, maxLogs=55 2024-12-06T14:43:57,400 INFO [RS:2;bb7ea3e46d9c:39099 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T14:43:57,405 INFO [RS:2;bb7ea3e46d9c:39099 {}] regionserver.HRegionServer(1482): Serving as bb7ea3e46d9c,39099,1733496237147, RpcServer on bb7ea3e46d9c/172.17.0.2:39099, sessionid=0x101ab19595c0003 2024-12-06T14:43:57,405 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T14:43:57,405 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bb7ea3e46d9c,39099,1733496237147 2024-12-06T14:43:57,405 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bb7ea3e46d9c,39099,1733496237147' 2024-12-06T14:43:57,405 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T14:43:57,408 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T14:43:57,409 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T14:43:57,409 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T14:43:57,409 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bb7ea3e46d9c,39099,1733496237147 2024-12-06T14:43:57,409 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bb7ea3e46d9c,39099,1733496237147' 2024-12-06T14:43:57,409 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T14:43:57,410 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T14:43:57,411 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T14:43:57,411 INFO [RS:2;bb7ea3e46d9c:39099 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T14:43:57,411 INFO [RS:2;bb7ea3e46d9c:39099 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T14:43:57,448 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,41829,1733496233612/bb7ea3e46d9c%2C41829%2C1733496233612.meta.1733496237387.meta 2024-12-06T14:43:57,456 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T14:43:57,456 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-06T14:43:57,457 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-06T14:43:57,457 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-06T14:43:57,457 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-06T14:43:57,457 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-06T14:43:57,457 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T14:43:57,457 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-06T14:43:57,457 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-06T14:43:57,458 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-06T14:43:57,459 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T14:43:57,459 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:57,460 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/info/c4da72a4eeae4da18a09d3cb081c0d15, delay=938 ms 2024-12-06T14:43:57,463 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/info/c4da72a4eeae4da18a09d3cb081c0d15 2024-12-06T14:43:57,463 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:57,464 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-06T14:43:57,464 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T14:43:57,464 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:57,469 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/ns/9165a473c2914db989931fcff2ea5665, delay=940 ms 2024-12-06T14:43:57,470 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/ns/9165a473c2914db989931fcff2ea5665 2024-12-06T14:43:57,470 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:57,471 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-06T14:43:57,471 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T14:43:57,471 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:57,471 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:57,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-06T14:43:57,472 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T14:43:57,472 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T14:43:57,474 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/table/0aa4aebf6f524063a4788826a2aa768f, delay=925 ms 2024-12-06T14:43:57,475 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/table/0aa4aebf6f524063a4788826a2aa768f 2024-12-06T14:43:57,475 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T14:43:57,476 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T14:43:57,477 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740 2024-12-06T14:43:57,477 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740 2024-12-06T14:43:57,477 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T14:43:57,478 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T14:43:57,478 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T14:43:57,479 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T14:43:57,480 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69083269, jitterRate=0.029420927166938782}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T14:43:57,480 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-06T14:43:57,481 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733496237457Writing region info on filesystem at 1733496237457Initializing all the Stores at 1733496237458 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496237458Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496237458Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733496237458Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733496237458Cleaning up temporary data from old regions at 1733496237478 (+20 ms)Running coprocessor post-open hooks at 1733496237480 (+2 ms)Region opened successfully at 1733496237481 (+1 ms) 2024-12-06T14:43:57,483 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1733496237323 2024-12-06T14:43:57,486 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=bb7ea3e46d9c,41829,1733496233612 2024-12-06T14:43:57,487 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bb7ea3e46d9c,41829,1733496233612, state=OPEN 2024-12-06T14:43:57,488 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-06T14:43:57,488 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-06T14:43:57,494 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42883 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=bb7ea3e46d9c,41829,1733496233612, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-06T14:43:57,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T14:43:57,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39099-0x101ab19595c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T14:43:57,501 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:43:57,501 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:43:57,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T14:43:57,501 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T14:43:57,501 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=bb7ea3e46d9c,41829,1733496233612 2024-12-06T14:43:57,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-06T14:43:57,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bb7ea3e46d9c,41829,1733496233612 in 332 msec 2024-12-06T14:43:57,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-06T14:43:57,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 517 msec 2024-12-06T14:43:57,508 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): bb7ea3e46d9c,41371,1733496233659 had 2 regions 2024-12-06T14:43:57,509 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure bb7ea3e46d9c,41371,1733496233659, splitWal=true, meta=true, isMeta: false 2024-12-06T14:43:57,510 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,41371,1733496233659-splitting dir is empty, no logs to split. 2024-12-06T14:43:57,510 INFO [PEWorker-2 {}] master.SplitWALManager(105): bb7ea3e46d9c,41371,1733496233659 WAL count=0, meta=false 2024-12-06T14:43:57,511 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,41371,1733496233659-splitting dir is empty, no logs to split. 2024-12-06T14:43:57,511 INFO [RS:2;bb7ea3e46d9c:39099 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T14:43:57,511 INFO [PEWorker-2 {}] master.SplitWALManager(105): bb7ea3e46d9c,41371,1733496233659 WAL count=0, meta=false 2024-12-06T14:43:57,511 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if bb7ea3e46d9c,41371,1733496233659 WAL splitting is done? wals=0, meta=false 2024-12-06T14:43:57,512 INFO [RS:2;bb7ea3e46d9c:39099 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bb7ea3e46d9c%2C39099%2C1733496237147, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,39099,1733496237147, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/oldWALs, maxLogs=55 2024-12-06T14:43:57,513 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733496234931, region=e3343eea25c5bc5cb9c4ce62f90e1fd6, ASSIGN}] 2024-12-06T14:43:57,515 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733496234931, region=e3343eea25c5bc5cb9c4ce62f90e1fd6, ASSIGN 2024-12-06T14:43:57,516 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733496234931, region=e3343eea25c5bc5cb9c4ce62f90e1fd6, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-06T14:43:57,542 INFO [RS:2;bb7ea3e46d9c:39099 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/WALs/bb7ea3e46d9c,39099,1733496237147/bb7ea3e46d9c%2C39099%2C1733496237147.1733496237513 2024-12-06T14:43:57,542 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T14:43:57,666 DEBUG [bb7ea3e46d9c:42883 {}] balancer.BalancerClusterState(204): Hosts are {bb7ea3e46d9c=0} racks are {/default-rack=0} 2024-12-06T14:43:57,666 DEBUG [bb7ea3e46d9c:42883 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T14:43:57,667 DEBUG [bb7ea3e46d9c:42883 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T14:43:57,667 DEBUG [bb7ea3e46d9c:42883 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T14:43:57,667 DEBUG [bb7ea3e46d9c:42883 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T14:43:57,667 INFO [bb7ea3e46d9c:42883 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T14:43:57,667 INFO [bb7ea3e46d9c:42883 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T14:43:57,667 DEBUG [bb7ea3e46d9c:42883 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T14:43:57,667 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e3343eea25c5bc5cb9c4ce62f90e1fd6, regionState=OPENING, regionLocation=bb7ea3e46d9c,39099,1733496237147 2024-12-06T14:43:57,668 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bb7ea3e46d9c:41371 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bb7ea3e46d9c/172.17.0.2:41371 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T14:43:57,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,41371,1733496233659, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,41371,1733496233659, seqNum=-1, error=java.net.ConnectException: Call to address=bb7ea3e46d9c:41371 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bb7ea3e46d9c/172.17.0.2:41371 2024-12-06T14:43:57,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,41371,1733496233659, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-06T14:43:57,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,41371,1733496233659, seqNum=-1 from cache 2024-12-06T14:43:57,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address bb7ea3e46d9c:41371 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bb7ea3e46d9c/172.17.0.2:41371 2024-12-06T14:43:57,774 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T14:43:57,774 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,41829,1733496233612, seqNum=-1] 2024-12-06T14:43:57,774 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T14:43:57,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49511, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T14:43:57,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733496234931, region=e3343eea25c5bc5cb9c4ce62f90e1fd6, ASSIGN because future has completed 2024-12-06T14:43:57,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e3343eea25c5bc5cb9c4ce62f90e1fd6, server=bb7ea3e46d9c,39099,1733496237147}] 2024-12-06T14:43:57,905 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-06T14:43:57,905 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T14:43:57,905 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T14:43:57,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:57,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:57,906 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T14:43:57,906 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-06T14:43:57,906 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=819277990, stopped=false 2024-12-06T14:43:57,906 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=bb7ea3e46d9c,42883,1733496233416 2024-12-06T14:43:57,933 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T14:43:57,935 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43497, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T14:43:57,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39099-0x101ab19595c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T14:43:57,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39099-0x101ab19595c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:57,938 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T14:43:57,938 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T14:43:57,939 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T14:43:57,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:57,939 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39099-0x101ab19595c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T14:43:57,939 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bb7ea3e46d9c,41829,1733496233612' ***** 2024-12-06T14:43:57,939 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-06T14:43:57,939 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bb7ea3e46d9c,39099,1733496237147' ***** 2024-12-06T14:43:57,939 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-06T14:43:57,939 INFO [RS:0;bb7ea3e46d9c:41829 {}] bucket.BucketCache(1738): Disabling cache 2024-12-06T14:43:57,939 INFO [RS:0;bb7ea3e46d9c:41829 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30/bucket.persistence 2024-12-06T14:43:57,939 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-06T14:43:57,939 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-06T14:43:57,939 INFO [RS:2;bb7ea3e46d9c:39099 {}] bucket.BucketCache(1738): Disabling cache 2024-12-06T14:43:57,940 INFO [RS:2;bb7ea3e46d9c:39099 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5789e2bf-f118-ec41-c2e6-69d52a52ef30/bucket.persistence 2024-12-06T14:43:57,940 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-06T14:43:57,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T14:43:57,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:57,942 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-06T14:43:57,940 DEBUG [RSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(271): Request to bb7ea3e46d9c,39099,1733496237147 failed, try=0 org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=bb7ea3e46d9c:39099 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:405) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-12-06T14:43:57,942 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-06T14:43:57,942 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-06T14:43:57,942 INFO [RS:2;bb7ea3e46d9c:39099 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-06T14:43:57,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T14:43:57,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:57,942 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-06T14:43:57,942 WARN [RSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(317): request to bb7ea3e46d9c,39099,1733496237147 failed due to org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=bb7ea3e46d9c:39099 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed, try=0, retrying... , request params: open_region { open_info { region { region_id: 1733496234937 table_name { namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733496234931" } start_key: "" end_key: "" offline: false split: false replica_id: 0 } open_proc_id: 11 } serverStartCode: 1733496237147 initiating_master_active_time: 1733496233754 master_system_time: 1733496237933 } 2024-12-06T14:43:57,943 INFO [RS:0;bb7ea3e46d9c:41829 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-06T14:43:57,943 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-06T14:43:57,943 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T14:43:57,943 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T14:43:57,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39099 {}] ipc.CallRunner(93): RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39099: skipped callId: 16 service: AdminService methodName: ExecuteProcedures size: 136 connection: 172.17.0.2:43497 deadline: 1733496297936 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$ExecuteProceduresRequest connection: 172.17.0.2:43497 2024-12-06T14:43:57,951 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T14:43:57,952 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] bucket.BucketCache(1487): Thread RS:2;bb7ea3e46d9c:39099 started persisting bucket cache to file 2024-12-06T14:43:57,953 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T14:43:57,953 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T14:43:57,953 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] bucket.BucketCache(1487): Thread RS:0;bb7ea3e46d9c:41829 started persisting bucket cache to file 2024-12-06T14:43:57,953 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T14:43:57,953 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T14:43:57,958 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-06T14:43:57,958 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-06T14:43:57,958 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733496237953 2024-12-06T14:43:57,958 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733496237953 2024-12-06T14:43:57,958 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] bucket.BucketCache(1509): Thread RS:0;bb7ea3e46d9c:41829 finished persisting bucket cache to file, renaming 2024-12-06T14:43:57,958 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] bucket.BucketCache(1509): Thread RS:2;bb7ea3e46d9c:39099 finished persisting bucket cache to file, renaming 2024-12-06T14:43:57,958 WARN [RS:2;bb7ea3e46d9c:39099 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-06T14:43:57,958 INFO [RS:0;bb7ea3e46d9c:41829 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T14:43:57,958 INFO [RS:2;bb7ea3e46d9c:39099 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T14:43:57,958 INFO [RS:2;bb7ea3e46d9c:39099 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T14:43:57,958 INFO [RS:0;bb7ea3e46d9c:41829 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T14:43:57,958 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T14:43:57,958 INFO [RS:0;bb7ea3e46d9c:41829 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T14:43:57,958 INFO [RS:2;bb7ea3e46d9c:39099 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T14:43:57,958 INFO [RS:2;bb7ea3e46d9c:39099 {}] regionserver.HRegionServer(959): stopping server bb7ea3e46d9c,39099,1733496237147 2024-12-06T14:43:57,958 INFO [RS:0;bb7ea3e46d9c:41829 {}] regionserver.HRegionServer(959): stopping server bb7ea3e46d9c,41829,1733496233612 2024-12-06T14:43:57,958 INFO [RS:2;bb7ea3e46d9c:39099 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T14:43:57,958 INFO [RS:0;bb7ea3e46d9c:41829 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T14:43:57,958 INFO [RS:2;bb7ea3e46d9c:39099 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;bb7ea3e46d9c:39099. 2024-12-06T14:43:57,958 INFO [RS:0;bb7ea3e46d9c:41829 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;bb7ea3e46d9c:41829. 2024-12-06T14:43:57,958 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T14:43:57,958 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T14:43:57,958 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:57,959 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:57,959 INFO [RS:2;bb7ea3e46d9c:39099 {}] regionserver.HRegionServer(976): stopping server bb7ea3e46d9c,39099,1733496237147; all regions closed. 2024-12-06T14:43:57,959 INFO [RS:0;bb7ea3e46d9c:41829 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T14:43:57,960 INFO [RS:0;bb7ea3e46d9c:41829 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T14:43:57,960 INFO [RS:0;bb7ea3e46d9c:41829 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T14:43:57,960 INFO [RS:0;bb7ea3e46d9c:41829 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-06T14:43:57,961 INFO [RS:0;bb7ea3e46d9c:41829 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-06T14:43:57,961 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-06T14:43:57,961 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T14:43:57,963 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/oldWALs 2024-12-06T14:43:57,963 INFO [RS:2;bb7ea3e46d9c:39099 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bb7ea3e46d9c%2C39099%2C1733496237147:(num 1733496237513) 2024-12-06T14:43:57,963 DEBUG [RS:2;bb7ea3e46d9c:39099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:57,963 INFO [regionserver/bb7ea3e46d9c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T14:43:57,963 INFO [RS:2;bb7ea3e46d9c:39099 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T14:43:57,963 INFO [RS:2;bb7ea3e46d9c:39099 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T14:43:57,963 INFO [RS:2;bb7ea3e46d9c:39099 {}] hbase.ChoreService(370): Chore service for: regionserver/bb7ea3e46d9c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T14:43:57,963 INFO [RS:2;bb7ea3e46d9c:39099 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T14:43:57,963 INFO [RS:2;bb7ea3e46d9c:39099 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T14:43:57,963 INFO [regionserver/bb7ea3e46d9c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T14:43:57,963 INFO [RS:2;bb7ea3e46d9c:39099 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T14:43:57,963 INFO [RS:2;bb7ea3e46d9c:39099 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T14:43:57,963 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-06T14:43:57,963 INFO [RS:2;bb7ea3e46d9c:39099 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39099 2024-12-06T14:43:57,963 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T14:43:57,964 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T14:43:57,964 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T14:43:57,964 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T14:43:57,964 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T14:43:57,964 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=485 B heapSize=1.70 KB 2024-12-06T14:43:57,973 INFO [regionserver/bb7ea3e46d9c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T14:43:57,976 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 17c4ba2c866f4fc08a504dadd1005a88 2024-12-06T14:43:57,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39099-0x101ab19595c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bb7ea3e46d9c,39099,1733496237147 2024-12-06T14:43:57,985 INFO [RS:2;bb7ea3e46d9c:39099 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T14:43:57,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T14:43:57,985 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$249/0x00007f1d1c477b20@512d539f rejected from java.util.concurrent.ThreadPoolExecutor@142fc7de[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 4] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-06T14:43:57,986 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bb7ea3e46d9c,39099,1733496237147] 2024-12-06T14:43:57,992 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/info/17c4ba2c866f4fc08a504dadd1005a88 is 209, key is testBlockEvictionOnGracefulStop1733496234931,,1733496234937.e3343eea25c5bc5cb9c4ce62f90e1fd6./info:regioninfo/1733496237667/Put/seqid=0 2024-12-06T14:43:57,993 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=485 B at sequenceid=18 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/info/17c4ba2c866f4fc08a504dadd1005a88 2024-12-06T14:43:57,996 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/.tmp/info/17c4ba2c866f4fc08a504dadd1005a88 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/info/17c4ba2c866f4fc08a504dadd1005a88 2024-12-06T14:43:57,997 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/info/17c4ba2c866f4fc08a504dadd1005a88, delay=925 ms 2024-12-06T14:43:57,999 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/info/17c4ba2c866f4fc08a504dadd1005a88, entries=3, sequenceid=18, filesize=5.9 K 2024-12-06T14:43:58,000 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~485 B/485, heapSize ~952 B/952, currentSize=0 B/0 for 1588230740 in 36ms, sequenceid=18, compaction requested=false 2024-12-06T14:43:58,004 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/info/c4da72a4eeae4da18a09d3cb081c0d15 2024-12-06T14:43:58,005 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/info/17c4ba2c866f4fc08a504dadd1005a88 2024-12-06T14:43:58,005 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bb7ea3e46d9c,39099,1733496237147 already deleted, retry=false 2024-12-06T14:43:58,005 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bb7ea3e46d9c,39099,1733496237147 expired; onlineServers=1 2024-12-06T14:43:58,007 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/ns/9165a473c2914db989931fcff2ea5665 2024-12-06T14:43:58,010 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/table/0aa4aebf6f524063a4788826a2aa768f 2024-12-06T14:43:58,023 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/data/hbase/meta/1588230740/recovered.edits/21.seqid, newMaxSeqId=21, maxSeqId=14 2024-12-06T14:43:58,024 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T14:43:58,024 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T14:43:58,024 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733496237961Running coprocessor pre-close hooks at 1733496237963 (+2 ms)Disabling compacts and flushes for region at 1733496237963Disabling writes for close at 1733496237964 (+1 ms)Obtaining lock to block concurrent updates at 1733496237964Preparing flush snapshotting stores in 1588230740 at 1733496237964Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=485, getHeapSize=1672, getOffHeapSize=0, getCellsCount=3 at 1733496237965 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733496237965Flushing 1588230740/info: creating writer at 1733496237965Flushing 1588230740/info: appending metadata at 1733496237991 (+26 ms)Flushing 1588230740/info: closing flushed file at 1733496237991Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6460969b: reopening flushed file at 1733496237996 (+5 ms)Finished flush of dataSize ~485 B/485, heapSize ~952 B/952, currentSize=0 B/0 for 1588230740 in 36ms, sequenceid=18, compaction requested=false at 1733496238000 (+4 ms)Writing region close event to WAL at 1733496238012 (+12 ms)Running coprocessor post-close hooks at 1733496238024 (+12 ms)Closed at 1733496238024 2024-12-06T14:43:58,025 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-06T14:43:58,044 DEBUG [RSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(271): Request to bb7ea3e46d9c,39099,1733496237147 failed, try=1 org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=bb7ea3e46d9c:39099 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$Stub.executeProcedures(AdminProtos.java:35059) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.lambda$executeProcedures$20(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:103) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:121) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.executeProcedures(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 11 more 2024-12-06T14:43:58,046 WARN [RSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(288): bb7ea3e46d9c,39099,1733496237147 tells us DoNotRetry due to org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=bb7ea3e46d9c:39099 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException, try=1, give up 2024-12-06T14:43:58,046 DEBUG [RSProcedureDispatcher-pool-1 {}] assignment.RegionRemoteProcedureBase(128): pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e3343eea25c5bc5cb9c4ce62f90e1fd6, server=bb7ea3e46d9c,39099,1733496237147 for region state=OPENING, location=bb7ea3e46d9c,39099,1733496237147, table=testBlockEvictionOnGracefulStop1733496234931, region=e3343eea25c5bc5cb9c4ce62f90e1fd6, targetServer bb7ea3e46d9c,39099,1733496237147 is dead, SCP will interrupt us, give up 2024-12-06T14:43:58,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39099-0x101ab19595c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T14:43:58,095 INFO [RS:2;bb7ea3e46d9c:39099 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T14:43:58,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39099-0x101ab19595c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T14:43:58,096 INFO [RS:2;bb7ea3e46d9c:39099 {}] regionserver.HRegionServer(1031): Exiting; stopping=bb7ea3e46d9c,39099,1733496237147; zookeeper connection closed. 2024-12-06T14:43:58,096 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@52a58885 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@52a58885 2024-12-06T14:43:58,161 INFO [RS:0;bb7ea3e46d9c:41829 {}] regionserver.HRegionServer(976): stopping server bb7ea3e46d9c,41829,1733496233612; all regions closed. 2024-12-06T14:43:58,164 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/oldWALs 2024-12-06T14:43:58,164 INFO [RS:0;bb7ea3e46d9c:41829 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bb7ea3e46d9c%2C41829%2C1733496233612.meta:.meta(num 1733496237387) 2024-12-06T14:43:58,166 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/oldWALs 2024-12-06T14:43:58,166 INFO [RS:0;bb7ea3e46d9c:41829 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bb7ea3e46d9c%2C41829%2C1733496233612:(num 1733496234504) 2024-12-06T14:43:58,166 DEBUG [RS:0;bb7ea3e46d9c:41829 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T14:43:58,166 INFO [RS:0;bb7ea3e46d9c:41829 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T14:43:58,167 INFO [RS:0;bb7ea3e46d9c:41829 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T14:43:58,167 INFO [RS:0;bb7ea3e46d9c:41829 {}] hbase.ChoreService(370): Chore service for: regionserver/bb7ea3e46d9c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T14:43:58,167 INFO [RS:0;bb7ea3e46d9c:41829 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T14:43:58,167 INFO [regionserver/bb7ea3e46d9c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T14:43:58,167 INFO [RS:0;bb7ea3e46d9c:41829 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41829 2024-12-06T14:43:58,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bb7ea3e46d9c,41829,1733496233612 2024-12-06T14:43:58,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T14:43:58,180 INFO [RS:0;bb7ea3e46d9c:41829 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T14:43:58,180 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bb7ea3e46d9c,41829,1733496233612] 2024-12-06T14:43:58,201 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bb7ea3e46d9c,41829,1733496233612 already deleted, retry=false 2024-12-06T14:43:58,201 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bb7ea3e46d9c,41829,1733496233612 expired; onlineServers=0 2024-12-06T14:43:58,201 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'bb7ea3e46d9c,42883,1733496233416' ***** 2024-12-06T14:43:58,201 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-06T14:43:58,201 INFO [M:0;bb7ea3e46d9c:42883 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T14:43:58,201 INFO [M:0;bb7ea3e46d9c:42883 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T14:43:58,201 DEBUG [M:0;bb7ea3e46d9c:42883 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-06T14:43:58,201 DEBUG [M:0;bb7ea3e46d9c:42883 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-06T14:43:58,201 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-06T14:43:58,201 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.large.0-1733496234173 {}] cleaner.HFileCleaner(306): Exit Thread[master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.large.0-1733496234173,5,FailOnTimeoutGroup] 2024-12-06T14:43:58,201 INFO [M:0;bb7ea3e46d9c:42883 {}] hbase.ChoreService(370): Chore service for: master/bb7ea3e46d9c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-06T14:43:58,202 INFO [M:0;bb7ea3e46d9c:42883 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T14:43:58,202 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.small.0-1733496234174 {}] cleaner.HFileCleaner(306): Exit Thread[master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.small.0-1733496234174,5,FailOnTimeoutGroup] 2024-12-06T14:43:58,202 DEBUG [M:0;bb7ea3e46d9c:42883 {}] master.HMaster(1795): Stopping service threads 2024-12-06T14:43:58,202 INFO [M:0;bb7ea3e46d9c:42883 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-06T14:43:58,202 INFO [M:0;bb7ea3e46d9c:42883 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T14:43:58,202 INFO [M:0;bb7ea3e46d9c:42883 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-06T14:43:58,202 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-06T14:43:58,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T14:43:58,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T14:43:58,216 DEBUG [M:0;bb7ea3e46d9c:42883 {}] zookeeper.ZKUtil(347): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-06T14:43:58,216 WARN [M:0;bb7ea3e46d9c:42883 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-06T14:43:58,217 INFO [M:0;bb7ea3e46d9c:42883 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/.lastflushedseqids 2024-12-06T14:43:58,227 INFO [M:0;bb7ea3e46d9c:42883 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-06T14:43:58,227 INFO [M:0;bb7ea3e46d9c:42883 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-06T14:43:58,227 DEBUG [M:0;bb7ea3e46d9c:42883 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T14:43:58,227 INFO [M:0;bb7ea3e46d9c:42883 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:43:58,227 DEBUG [M:0;bb7ea3e46d9c:42883 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:43:58,227 DEBUG [M:0;bb7ea3e46d9c:42883 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T14:43:58,227 DEBUG [M:0;bb7ea3e46d9c:42883 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:43:58,227 INFO [M:0;bb7ea3e46d9c:42883 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.94 KB heapSize=44.16 KB 2024-12-06T14:43:58,236 DEBUG [M:0;bb7ea3e46d9c:42883 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 762b1192d8654645808430c4e4aa6267 2024-12-06T14:43:58,253 DEBUG [M:0;bb7ea3e46d9c:42883 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/762b1192d8654645808430c4e4aa6267 is 82, key is hbase:meta,,1/info:regioninfo/1733496237486/Put/seqid=0 2024-12-06T14:43:58,254 INFO [M:0;bb7ea3e46d9c:42883 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/762b1192d8654645808430c4e4aa6267 2024-12-06T14:43:58,261 DEBUG [M:0;bb7ea3e46d9c:42883 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1c623c2047c94b2fb78e5752e2f816eb 2024-12-06T14:43:58,275 DEBUG [M:0;bb7ea3e46d9c:42883 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1c623c2047c94b2fb78e5752e2f816eb is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733496235328/Put/seqid=0 2024-12-06T14:43:58,276 INFO [M:0;bb7ea3e46d9c:42883 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.71 KB at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1c623c2047c94b2fb78e5752e2f816eb 2024-12-06T14:43:58,283 DEBUG [M:0;bb7ea3e46d9c:42883 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 85659e164242484b9956ab303d853f43 2024-12-06T14:43:58,291 INFO [RS:0;bb7ea3e46d9c:41829 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T14:43:58,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T14:43:58,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101ab19595c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T14:43:58,291 INFO [RS:0;bb7ea3e46d9c:41829 {}] regionserver.HRegionServer(1031): Exiting; stopping=bb7ea3e46d9c,41829,1733496233612; zookeeper connection closed. 2024-12-06T14:43:58,291 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@dfce73a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@dfce73a 2024-12-06T14:43:58,291 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-06T14:43:58,297 DEBUG [M:0;bb7ea3e46d9c:42883 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/85659e164242484b9956ab303d853f43 is 69, key is bb7ea3e46d9c,39099,1733496237147/rs:state/1733496237268/Put/seqid=0 2024-12-06T14:43:58,298 INFO [M:0;bb7ea3e46d9c:42883 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/85659e164242484b9956ab303d853f43 2024-12-06T14:43:58,300 INFO [M:0;bb7ea3e46d9c:42883 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 85659e164242484b9956ab303d853f43 2024-12-06T14:43:58,300 DEBUG [M:0;bb7ea3e46d9c:42883 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/762b1192d8654645808430c4e4aa6267 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/762b1192d8654645808430c4e4aa6267 2024-12-06T14:43:58,302 INFO [M:0;bb7ea3e46d9c:42883 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/762b1192d8654645808430c4e4aa6267, entries=14, sequenceid=90, filesize=5.9 K 2024-12-06T14:43:58,302 DEBUG [M:0;bb7ea3e46d9c:42883 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1c623c2047c94b2fb78e5752e2f816eb as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1c623c2047c94b2fb78e5752e2f816eb 2024-12-06T14:43:58,304 INFO [M:0;bb7ea3e46d9c:42883 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1c623c2047c94b2fb78e5752e2f816eb, entries=11, sequenceid=90, filesize=7.8 K 2024-12-06T14:43:58,304 DEBUG [M:0;bb7ea3e46d9c:42883 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/85659e164242484b9956ab303d853f43 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/85659e164242484b9956ab303d853f43 2024-12-06T14:43:58,306 INFO [M:0;bb7ea3e46d9c:42883 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 85659e164242484b9956ab303d853f43 2024-12-06T14:43:58,306 INFO [M:0;bb7ea3e46d9c:42883 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0455f73c-a31d-ae91-5549-8329bf9b63fb/0dfb946a-2c13-230a-6845-38a6b7704b4f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/85659e164242484b9956ab303d853f43, entries=3, sequenceid=90, filesize=5.3 K 2024-12-06T14:43:58,308 INFO [M:0;bb7ea3e46d9c:42883 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.94 KB/35780, heapSize ~43.87 KB/44920, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 80ms, sequenceid=90, compaction requested=false 2024-12-06T14:43:58,312 INFO [M:0;bb7ea3e46d9c:42883 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T14:43:58,312 DEBUG [M:0;bb7ea3e46d9c:42883 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733496238227Disabling compacts and flushes for region at 1733496238227Disabling writes for close at 1733496238227Obtaining lock to block concurrent updates at 1733496238227Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733496238227Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35780, getHeapSize=45160, getOffHeapSize=0, getCellsCount=111 at 1733496238228 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733496238228Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733496238229 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733496238252 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733496238253 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733496238256 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733496238274 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733496238274Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733496238278 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733496238296 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733496238296Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9fbbb3b: reopening flushed file at 1733496238300 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61159bd9: reopening flushed file at 1733496238302 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26949015: reopening flushed file at 1733496238304 (+2 ms)Finished flush of dataSize ~34.94 KB/35780, heapSize ~43.87 KB/44920, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 80ms, sequenceid=90, compaction requested=false at 1733496238308 (+4 ms)Writing region close event to WAL at 1733496238310 (+2 ms)Closed at 1733496238310 2024-12-06T14:43:58,314 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T14:43:58,314 INFO [M:0;bb7ea3e46d9c:42883 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-06T14:43:58,315 INFO [M:0;bb7ea3e46d9c:42883 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42883 2024-12-06T14:43:58,315 INFO [M:0;bb7ea3e46d9c:42883 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T14:43:58,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T14:43:58,427 INFO [M:0;bb7ea3e46d9c:42883 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T14:43:58,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42883-0x101ab19595c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T14:43:58,430 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-06T14:43:58,431 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-06T14:43:58,431 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T14:43:58,439 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 49) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=612 (was 586) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6394 (was 6479)