2024-12-11 03:46:04,120 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-11 03:46:04,132 main DEBUG Took 0.009554 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-11 03:46:04,132 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-11 03:46:04,132 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-11 03:46:04,133 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-11 03:46:04,134 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 03:46:04,141 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-11 03:46:04,162 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 03:46:04,164 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 03:46:04,165 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 03:46:04,165 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 03:46:04,166 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 03:46:04,166 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 03:46:04,167 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 03:46:04,168 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 03:46:04,168 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 03:46:04,169 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 03:46:04,170 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 03:46:04,170 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 03:46:04,171 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 03:46:04,172 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 03:46:04,172 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 03:46:04,172 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 03:46:04,173 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 03:46:04,173 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 03:46:04,174 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 03:46:04,174 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 03:46:04,175 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 03:46:04,175 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 03:46:04,176 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 03:46:04,176 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 03:46:04,177 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 03:46:04,177 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-11 03:46:04,179 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 03:46:04,180 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-11 03:46:04,182 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-11 03:46:04,182 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-11 03:46:04,184 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-11 03:46:04,185 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-11 03:46:04,195 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-11 03:46:04,198 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-11 03:46:04,199 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-11 03:46:04,200 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-11 03:46:04,200 main DEBUG createAppenders(={Console}) 2024-12-11 03:46:04,201 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-11 03:46:04,201 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-11 03:46:04,202 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-11 03:46:04,202 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-11 03:46:04,203 main DEBUG OutputStream closed 2024-12-11 03:46:04,203 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-11 03:46:04,204 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-11 03:46:04,204 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-11 03:46:04,289 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-11 03:46:04,291 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-11 03:46:04,292 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-11 03:46:04,294 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-11 03:46:04,294 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-11 03:46:04,295 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-11 03:46:04,295 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-11 03:46:04,296 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-11 03:46:04,296 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-11 03:46:04,297 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-11 03:46:04,297 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-11 03:46:04,298 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-11 03:46:04,298 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-11 03:46:04,298 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-11 03:46:04,299 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-11 03:46:04,299 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-11 03:46:04,299 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-11 03:46:04,300 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-11 03:46:04,302 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-11 03:46:04,303 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-11 03:46:04,303 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-11 03:46:04,304 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-11T03:46:04,318 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-11 03:46:04,321 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-11 03:46:04,321 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-11T03:46:04,553 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511 2024-12-11T03:46:04,582 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=165, ProcessCount=11, AvailableMemoryMB=4544 2024-12-11T03:46:04,677 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-11T03:46:04,764 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:04,780 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/cluster_ecd99d3d-dcff-3d48-7cb8-e34a854cf6e2, deleteOnExit=true 2024-12-11T03:46:04,875 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/cluster_ecd99d3d-dcff-3d48-7cb8-e34a854cf6e2/zookeeper_0, clientPort=57764, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/cluster_ecd99d3d-dcff-3d48-7cb8-e34a854cf6e2/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/cluster_ecd99d3d-dcff-3d48-7cb8-e34a854cf6e2/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-11T03:46:04,887 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57764 2024-12-11T03:46:04,888 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:04,972 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e with version=8 2024-12-11T03:46:04,973 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/hbase-staging 2024-12-11T03:46:05,066 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-11T03:46:05,334 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c49966d8a95f:0 server-side Connection retries=45 2024-12-11T03:46:05,344 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:05,346 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:05,353 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T03:46:05,354 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:05,354 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T03:46:05,493 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-11T03:46:05,583 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T03:46:05,621 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-11T03:46:05,631 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-11T03:46:05,636 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T03:46:05,677 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 15127 (auto-detected) 2024-12-11T03:46:05,678 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-11T03:46:05,698 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33041 2024-12-11T03:46:05,722 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33041 connecting to ZooKeeper ensemble=127.0.0.1:57764 2024-12-11T03:46:05,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:330410x0, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T03:46:05,761 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33041-0x1019fa0d63e0000 connected 2024-12-11T03:46:05,790 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:05,791 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:05,799 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T03:46:05,803 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e, hbase.cluster.distributed=false 2024-12-11T03:46:05,828 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T03:46:05,835 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33041 2024-12-11T03:46:05,835 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33041 2024-12-11T03:46:05,836 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33041 2024-12-11T03:46:05,836 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33041 2024-12-11T03:46:05,836 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33041 2024-12-11T03:46:05,929 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c49966d8a95f:0 server-side Connection retries=45 2024-12-11T03:46:05,931 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:05,931 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:05,932 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T03:46:05,932 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:05,932 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T03:46:05,935 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-11T03:46:05,938 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T03:46:05,939 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40763 2024-12-11T03:46:05,941 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40763 connecting to ZooKeeper ensemble=127.0.0.1:57764 2024-12-11T03:46:05,941 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:05,942 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:05,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:407630x0, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T03:46:05,948 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40763-0x1019fa0d63e0001 connected 2024-12-11T03:46:05,948 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T03:46:05,955 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-11T03:46:05,968 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/bucket.cache 2024-12-11T03:46:05,976 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-11T03:46:05,978 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-11T03:46:05,980 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-11T03:46:05,982 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-11T03:46:05,982 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-11T03:46:05,990 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-11T03:46:05,990 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-11T03:46:05,991 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/bucket.persistence is complete. 2024-12-11T03:46:05,997 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-11T03:46:06,000 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-11T03:46:06,006 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T03:46:06,007 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40763 2024-12-11T03:46:06,007 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40763 2024-12-11T03:46:06,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40763 2024-12-11T03:46:06,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40763 2024-12-11T03:46:06,013 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40763 2024-12-11T03:46:06,031 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c49966d8a95f:0 server-side Connection retries=45 2024-12-11T03:46:06,031 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:06,031 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:06,032 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T03:46:06,032 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:06,032 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T03:46:06,032 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-11T03:46:06,033 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T03:46:06,034 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46637 2024-12-11T03:46:06,036 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46637 connecting to ZooKeeper ensemble=127.0.0.1:57764 2024-12-11T03:46:06,036 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:06,037 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:06,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:466370x0, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T03:46:06,042 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:466370x0, quorum=127.0.0.1:57764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T03:46:06,042 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46637-0x1019fa0d63e0002 connected 2024-12-11T03:46:06,043 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-11T03:46:06,044 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/bucket.cache 2024-12-11T03:46:06,045 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-11T03:46:06,045 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-11T03:46:06,045 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-11T03:46:06,046 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-11T03:46:06,046 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-11T03:46:06,047 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-11T03:46:06,047 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-11T03:46:06,048 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/bucket.persistence is complete. 2024-12-11T03:46:06,048 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-11T03:46:06,049 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46637-0x1019fa0d63e0002, quorum=127.0.0.1:57764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-11T03:46:06,051 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46637-0x1019fa0d63e0002, quorum=127.0.0.1:57764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T03:46:06,054 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46637 2024-12-11T03:46:06,055 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46637 2024-12-11T03:46:06,057 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46637 2024-12-11T03:46:06,058 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46637 2024-12-11T03:46:06,058 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46637 2024-12-11T03:46:06,076 DEBUG [M:0;c49966d8a95f:33041 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c49966d8a95f:33041 2024-12-11T03:46:06,077 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c49966d8a95f,33041,1733888765137 2024-12-11T03:46:06,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1019fa0d63e0002, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T03:46:06,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T03:46:06,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T03:46:06,090 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c49966d8a95f,33041,1733888765137 2024-12-11T03:46:06,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:06,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1019fa0d63e0002, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-11T03:46:06,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-11T03:46:06,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:06,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1019fa0d63e0002, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:06,115 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-11T03:46:06,117 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c49966d8a95f,33041,1733888765137 from backup master directory 2024-12-11T03:46:06,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c49966d8a95f,33041,1733888765137 2024-12-11T03:46:06,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T03:46:06,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1019fa0d63e0002, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T03:46:06,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T03:46:06,121 WARN [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T03:46:06,122 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c49966d8a95f,33041,1733888765137 2024-12-11T03:46:06,124 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-11T03:46:06,127 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-11T03:46:06,137 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/hbase.id] with ID: fd02e8ae-f0b8-4e5d-a2a1-b8508a87c1cf 2024-12-11T03:46:06,137 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/.tmp/hbase.id 2024-12-11T03:46:06,144 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/hbase.id] 2024-12-11T03:46:06,184 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:06,191 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-11T03:46:06,221 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 26ms. 2024-12-11T03:46:06,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:06,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:06,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1019fa0d63e0002, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:06,254 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T03:46:06,258 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-11T03:46:06,319 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T03:46:06,326 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T03:46:06,392 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store 2024-12-11T03:46:06,413 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-11T03:46:06,416 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T03:46:06,417 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-11T03:46:06,417 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T03:46:06,418 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T03:46:06,419 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-11T03:46:06,419 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T03:46:06,420 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T03:46:06,420 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733888766417Disabling compacts and flushes for region at 1733888766417Disabling writes for close at 1733888766419 (+2 ms)Writing region close event to WAL at 1733888766419Closed at 1733888766419 2024-12-11T03:46:06,421 WARN [master/c49966d8a95f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/.initializing 2024-12-11T03:46:06,421 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/WALs/c49966d8a95f,33041,1733888765137 2024-12-11T03:46:06,436 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T03:46:06,449 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c49966d8a95f%2C33041%2C1733888765137, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/WALs/c49966d8a95f,33041,1733888765137, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/oldWALs, maxLogs=10 2024-12-11T03:46:06,490 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/WALs/c49966d8a95f,33041,1733888765137/c49966d8a95f%2C33041%2C1733888765137.1733888766453 2024-12-11T03:46:06,492 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-11T03:46:06,492 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-11T03:46:06,493 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T03:46:06,498 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:06,499 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:06,542 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:06,575 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-11T03:46:06,579 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:06,581 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T03:46:06,582 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:06,588 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-11T03:46:06,588 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:06,589 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T03:46:06,589 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:06,594 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-11T03:46:06,595 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:06,595 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T03:46:06,595 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:06,600 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-11T03:46:06,601 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:06,601 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T03:46:06,602 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:06,606 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:06,607 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:06,611 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:06,612 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:06,616 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T03:46:06,619 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:06,627 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T03:46:06,628 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75140139, jitterRate=0.11967532336711884}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T03:46:06,634 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733888766514Initializing all the Stores at 1733888766515 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733888766515Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733888766516 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733888766517 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733888766517Cleaning up temporary data from old regions at 1733888766612 (+95 ms)Region opened successfully at 1733888766634 (+22 ms) 2024-12-11T03:46:06,636 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-11T03:46:06,669 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bce5d6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c49966d8a95f/172.17.0.2:0 2024-12-11T03:46:06,704 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-11T03:46:06,718 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-11T03:46:06,718 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-11T03:46:06,722 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T03:46:06,722 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T03:46:06,729 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-11T03:46:06,730 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-11T03:46:06,760 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-11T03:46:06,770 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-11T03:46:06,774 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-11T03:46:06,777 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-11T03:46:06,778 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-11T03:46:06,780 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-11T03:46:06,782 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-11T03:46:06,787 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-11T03:46:06,788 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-11T03:46:06,790 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-11T03:46:06,791 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-11T03:46:06,810 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-11T03:46:06,814 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-11T03:46:06,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T03:46:06,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T03:46:06,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1019fa0d63e0002, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T03:46:06,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1019fa0d63e0002, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:06,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:06,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:06,834 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c49966d8a95f,33041,1733888765137, sessionid=0x1019fa0d63e0000, setting cluster-up flag (Was=false) 2024-12-11T03:46:06,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:06,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:06,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1019fa0d63e0002, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:06,855 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-11T03:46:06,857 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c49966d8a95f,33041,1733888765137 2024-12-11T03:46:06,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:06,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1019fa0d63e0002, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:06,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:06,869 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-11T03:46:06,871 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c49966d8a95f,33041,1733888765137 2024-12-11T03:46:06,876 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-11T03:46:06,949 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T03:46:06,962 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-11T03:46:06,964 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.HRegionServer(746): ClusterId : fd02e8ae-f0b8-4e5d-a2a1-b8508a87c1cf 2024-12-11T03:46:06,964 INFO [RS:0;c49966d8a95f:40763 {}] regionserver.HRegionServer(746): ClusterId : fd02e8ae-f0b8-4e5d-a2a1-b8508a87c1cf 2024-12-11T03:46:06,967 DEBUG [RS:0;c49966d8a95f:40763 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-11T03:46:06,967 DEBUG [RS:1;c49966d8a95f:46637 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-11T03:46:06,971 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-11T03:46:06,974 DEBUG [RS:1;c49966d8a95f:46637 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-11T03:46:06,974 DEBUG [RS:0;c49966d8a95f:40763 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-11T03:46:06,974 DEBUG [RS:1;c49966d8a95f:46637 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-11T03:46:06,974 DEBUG [RS:0;c49966d8a95f:40763 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-11T03:46:06,978 DEBUG [RS:0;c49966d8a95f:40763 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-11T03:46:06,978 DEBUG [RS:1;c49966d8a95f:46637 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-11T03:46:06,979 DEBUG [RS:0;c49966d8a95f:40763 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a99c058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c49966d8a95f/172.17.0.2:0 2024-12-11T03:46:06,979 DEBUG [RS:1;c49966d8a95f:46637 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60253db2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c49966d8a95f/172.17.0.2:0 2024-12-11T03:46:06,978 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c49966d8a95f,33041,1733888765137 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-11T03:46:06,987 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c49966d8a95f:0, corePoolSize=5, maxPoolSize=5 2024-12-11T03:46:06,987 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c49966d8a95f:0, corePoolSize=5, maxPoolSize=5 2024-12-11T03:46:06,988 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c49966d8a95f:0, corePoolSize=5, maxPoolSize=5 2024-12-11T03:46:06,988 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c49966d8a95f:0, corePoolSize=5, maxPoolSize=5 2024-12-11T03:46:06,988 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c49966d8a95f:0, corePoolSize=10, maxPoolSize=10 2024-12-11T03:46:06,988 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:06,989 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c49966d8a95f:0, corePoolSize=2, maxPoolSize=2 2024-12-11T03:46:06,989 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:06,994 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T03:46:06,995 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-11T03:46:06,995 DEBUG [RS:1;c49966d8a95f:46637 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;c49966d8a95f:46637 2024-12-11T03:46:06,995 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733888796995 2024-12-11T03:46:06,997 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-11T03:46:06,997 DEBUG [RS:0;c49966d8a95f:40763 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c49966d8a95f:40763 2024-12-11T03:46:06,998 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-11T03:46:06,999 INFO [RS:0;c49966d8a95f:40763 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-11T03:46:06,999 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-11T03:46:06,999 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-11T03:46:06,999 INFO [RS:0;c49966d8a95f:40763 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-11T03:46:06,999 DEBUG [RS:1;c49966d8a95f:46637 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-11T03:46:06,999 DEBUG [RS:0;c49966d8a95f:40763 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-11T03:46:07,001 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:07,001 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-11T03:46:07,002 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-11T03:46:07,003 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.HRegionServer(2659): reportForDuty to master=c49966d8a95f,33041,1733888765137 with port=46637, startcode=1733888766030 2024-12-11T03:46:07,003 INFO [RS:0;c49966d8a95f:40763 {}] regionserver.HRegionServer(2659): reportForDuty to master=c49966d8a95f,33041,1733888765137 with port=40763, startcode=1733888765890 2024-12-11T03:46:07,003 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-11T03:46:07,003 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-11T03:46:07,004 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-11T03:46:07,007 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,017 DEBUG [RS:0;c49966d8a95f:40763 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-11T03:46:07,017 DEBUG [RS:1;c49966d8a95f:46637 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-11T03:46:07,017 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-11T03:46:07,019 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-11T03:46:07,019 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-11T03:46:07,023 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-11T03:46:07,023 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-11T03:46:07,026 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c49966d8a95f:0:becomeActiveMaster-HFileCleaner.large.0-1733888767025,5,FailOnTimeoutGroup] 2024-12-11T03:46:07,031 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c49966d8a95f:0:becomeActiveMaster-HFileCleaner.small.0-1733888767026,5,FailOnTimeoutGroup] 2024-12-11T03:46:07,032 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,032 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-11T03:46:07,032 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-11T03:46:07,033 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,033 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e 2024-12-11T03:46:07,033 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,045 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T03:46:07,046 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-11T03:46:07,051 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-11T03:46:07,051 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:07,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T03:46:07,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-11T03:46:07,057 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-11T03:46:07,057 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:07,057 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T03:46:07,057 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-11T03:46:07,067 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-11T03:46:07,067 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:07,068 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T03:46:07,068 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-11T03:46:07,074 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-11T03:46:07,074 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:07,074 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T03:46:07,075 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-11T03:46:07,076 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740 2024-12-11T03:46:07,076 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740 2024-12-11T03:46:07,077 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-11T03:46:07,077 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-11T03:46:07,078 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T03:46:07,079 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-11T03:46:07,088 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T03:46:07,088 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63244464, jitterRate=-0.05758404731750488}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T03:46:07,093 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733888767045Initializing all the Stores at 1733888767045Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733888767045Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733888767046 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733888767046Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733888767046Cleaning up temporary data from old regions at 1733888767077 (+31 ms)Region opened successfully at 1733888767092 (+15 ms) 2024-12-11T03:46:07,093 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-11T03:46:07,093 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-11T03:46:07,093 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-11T03:46:07,093 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-11T03:46:07,094 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-11T03:46:07,095 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-11T03:46:07,095 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733888767093Disabling compacts and flushes for region at 1733888767093Disabling writes for close at 1733888767094 (+1 ms)Writing region close event to WAL at 1733888767095 (+1 ms)Closed at 1733888767095 2024-12-11T03:46:07,099 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T03:46:07,099 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-11T03:46:07,099 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34543, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-11T03:46:07,099 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36875, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-11T03:46:07,106 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33041 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c49966d8a95f,40763,1733888765890 2024-12-11T03:46:07,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-11T03:46:07,109 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33041 {}] master.ServerManager(517): Registering regionserver=c49966d8a95f,40763,1733888765890 2024-12-11T03:46:07,114 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T03:46:07,116 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-11T03:46:07,119 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33041 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c49966d8a95f,46637,1733888766030 2024-12-11T03:46:07,119 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33041 {}] master.ServerManager(517): Registering regionserver=c49966d8a95f,46637,1733888766030 2024-12-11T03:46:07,123 DEBUG [RS:1;c49966d8a95f:46637 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e 2024-12-11T03:46:07,123 DEBUG [RS:0;c49966d8a95f:40763 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e 2024-12-11T03:46:07,123 DEBUG [RS:0;c49966d8a95f:40763 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-11T03:46:07,123 DEBUG [RS:1;c49966d8a95f:46637 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-11T03:46:07,123 DEBUG [RS:0;c49966d8a95f:40763 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-11T03:46:07,123 DEBUG [RS:1;c49966d8a95f:46637 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-11T03:46:07,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T03:46:07,137 DEBUG [RS:0;c49966d8a95f:40763 {}] zookeeper.ZKUtil(111): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c49966d8a95f,40763,1733888765890 2024-12-11T03:46:07,137 DEBUG [RS:1;c49966d8a95f:46637 {}] zookeeper.ZKUtil(111): regionserver:46637-0x1019fa0d63e0002, quorum=127.0.0.1:57764, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c49966d8a95f,46637,1733888766030 2024-12-11T03:46:07,137 WARN [RS:0;c49966d8a95f:40763 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T03:46:07,137 WARN [RS:1;c49966d8a95f:46637 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T03:46:07,137 INFO [RS:0;c49966d8a95f:40763 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T03:46:07,137 INFO [RS:1;c49966d8a95f:46637 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T03:46:07,137 DEBUG [RS:0;c49966d8a95f:40763 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,40763,1733888765890 2024-12-11T03:46:07,138 DEBUG [RS:1;c49966d8a95f:46637 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,46637,1733888766030 2024-12-11T03:46:07,139 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c49966d8a95f,46637,1733888766030] 2024-12-11T03:46:07,139 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c49966d8a95f,40763,1733888765890] 2024-12-11T03:46:07,165 INFO [RS:0;c49966d8a95f:40763 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-11T03:46:07,165 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-11T03:46:07,178 INFO [RS:0;c49966d8a95f:40763 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-11T03:46:07,178 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-11T03:46:07,183 INFO [RS:0;c49966d8a95f:40763 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-11T03:46:07,183 INFO [RS:1;c49966d8a95f:46637 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-11T03:46:07,183 INFO [RS:0;c49966d8a95f:40763 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,183 INFO [RS:1;c49966d8a95f:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,184 INFO [RS:0;c49966d8a95f:40763 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-11T03:46:07,184 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-11T03:46:07,189 INFO [RS:0;c49966d8a95f:40763 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-11T03:46:07,189 INFO [RS:1;c49966d8a95f:46637 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-11T03:46:07,191 INFO [RS:0;c49966d8a95f:40763 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,191 INFO [RS:1;c49966d8a95f:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,191 DEBUG [RS:0;c49966d8a95f:40763 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,191 DEBUG [RS:1;c49966d8a95f:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,191 DEBUG [RS:0;c49966d8a95f:40763 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,192 DEBUG [RS:1;c49966d8a95f:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,192 DEBUG [RS:0;c49966d8a95f:40763 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,192 DEBUG [RS:0;c49966d8a95f:40763 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,192 DEBUG [RS:1;c49966d8a95f:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,192 DEBUG [RS:0;c49966d8a95f:40763 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,192 DEBUG [RS:1;c49966d8a95f:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,192 DEBUG [RS:0;c49966d8a95f:40763 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c49966d8a95f:0, corePoolSize=2, maxPoolSize=2 2024-12-11T03:46:07,192 DEBUG [RS:0;c49966d8a95f:40763 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,192 DEBUG [RS:1;c49966d8a95f:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,192 DEBUG [RS:0;c49966d8a95f:40763 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,192 DEBUG [RS:1;c49966d8a95f:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c49966d8a95f:0, corePoolSize=2, maxPoolSize=2 2024-12-11T03:46:07,193 DEBUG [RS:0;c49966d8a95f:40763 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,193 DEBUG [RS:1;c49966d8a95f:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,193 DEBUG [RS:0;c49966d8a95f:40763 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,193 DEBUG [RS:1;c49966d8a95f:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,193 DEBUG [RS:0;c49966d8a95f:40763 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,193 DEBUG [RS:1;c49966d8a95f:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,193 DEBUG [RS:0;c49966d8a95f:40763 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,193 DEBUG [RS:0;c49966d8a95f:40763 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c49966d8a95f:0, corePoolSize=3, maxPoolSize=3 2024-12-11T03:46:07,193 DEBUG [RS:1;c49966d8a95f:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,194 DEBUG [RS:0;c49966d8a95f:40763 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c49966d8a95f:0, corePoolSize=3, maxPoolSize=3 2024-12-11T03:46:07,194 DEBUG [RS:1;c49966d8a95f:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,194 DEBUG [RS:1;c49966d8a95f:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:07,194 DEBUG [RS:1;c49966d8a95f:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c49966d8a95f:0, corePoolSize=3, maxPoolSize=3 2024-12-11T03:46:07,194 DEBUG [RS:1;c49966d8a95f:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c49966d8a95f:0, corePoolSize=3, maxPoolSize=3 2024-12-11T03:46:07,195 INFO [RS:0;c49966d8a95f:40763 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,195 INFO [RS:0;c49966d8a95f:40763 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,195 INFO [RS:0;c49966d8a95f:40763 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,195 INFO [RS:0;c49966d8a95f:40763 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,195 INFO [RS:0;c49966d8a95f:40763 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,196 INFO [RS:0;c49966d8a95f:40763 {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,40763,1733888765890-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T03:46:07,196 INFO [RS:1;c49966d8a95f:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,196 INFO [RS:1;c49966d8a95f:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,196 INFO [RS:1;c49966d8a95f:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,196 INFO [RS:1;c49966d8a95f:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,197 INFO [RS:1;c49966d8a95f:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,197 INFO [RS:1;c49966d8a95f:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,46637,1733888766030-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T03:46:07,218 INFO [RS:0;c49966d8a95f:40763 {}] regionserver.HeapMemoryManager(213): Starting, tuneOn=false 2024-12-11T03:46:07,218 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.HeapMemoryManager(213): Starting, tuneOn=false 2024-12-11T03:46:07,220 INFO [RS:0;c49966d8a95f:40763 {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,40763,1733888765890-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,220 INFO [RS:1;c49966d8a95f:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,46637,1733888766030-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,221 INFO [RS:0;c49966d8a95f:40763 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,221 INFO [RS:1;c49966d8a95f:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,221 INFO [RS:0;c49966d8a95f:40763 {}] regionserver.Replication(171): c49966d8a95f,40763,1733888765890 started 2024-12-11T03:46:07,221 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.Replication(171): c49966d8a95f,46637,1733888766030 started 2024-12-11T03:46:07,241 INFO [RS:0;c49966d8a95f:40763 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,242 INFO [RS:0;c49966d8a95f:40763 {}] regionserver.HRegionServer(1482): Serving as c49966d8a95f,40763,1733888765890, RpcServer on c49966d8a95f/172.17.0.2:40763, sessionid=0x1019fa0d63e0001 2024-12-11T03:46:07,243 DEBUG [RS:0;c49966d8a95f:40763 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-11T03:46:07,243 DEBUG [RS:0;c49966d8a95f:40763 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c49966d8a95f,40763,1733888765890 2024-12-11T03:46:07,243 DEBUG [RS:0;c49966d8a95f:40763 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c49966d8a95f,40763,1733888765890' 2024-12-11T03:46:07,244 DEBUG [RS:0;c49966d8a95f:40763 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-11T03:46:07,245 DEBUG [RS:0;c49966d8a95f:40763 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-11T03:46:07,246 DEBUG [RS:0;c49966d8a95f:40763 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-11T03:46:07,246 DEBUG [RS:0;c49966d8a95f:40763 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-11T03:46:07,246 DEBUG [RS:0;c49966d8a95f:40763 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c49966d8a95f,40763,1733888765890 2024-12-11T03:46:07,246 DEBUG [RS:0;c49966d8a95f:40763 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c49966d8a95f,40763,1733888765890' 2024-12-11T03:46:07,246 DEBUG [RS:0;c49966d8a95f:40763 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-11T03:46:07,247 INFO [RS:1;c49966d8a95f:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,247 DEBUG [RS:0;c49966d8a95f:40763 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-11T03:46:07,247 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.HRegionServer(1482): Serving as c49966d8a95f,46637,1733888766030, RpcServer on c49966d8a95f/172.17.0.2:46637, sessionid=0x1019fa0d63e0002 2024-12-11T03:46:07,248 DEBUG [RS:1;c49966d8a95f:46637 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-11T03:46:07,248 DEBUG [RS:1;c49966d8a95f:46637 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c49966d8a95f,46637,1733888766030 2024-12-11T03:46:07,248 DEBUG [RS:0;c49966d8a95f:40763 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-11T03:46:07,248 DEBUG [RS:1;c49966d8a95f:46637 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c49966d8a95f,46637,1733888766030' 2024-12-11T03:46:07,248 DEBUG [RS:1;c49966d8a95f:46637 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-11T03:46:07,248 INFO [RS:0;c49966d8a95f:40763 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-11T03:46:07,248 INFO [RS:0;c49966d8a95f:40763 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-11T03:46:07,249 DEBUG [RS:1;c49966d8a95f:46637 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-11T03:46:07,249 DEBUG [RS:1;c49966d8a95f:46637 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-11T03:46:07,249 DEBUG [RS:1;c49966d8a95f:46637 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-11T03:46:07,250 DEBUG [RS:1;c49966d8a95f:46637 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c49966d8a95f,46637,1733888766030 2024-12-11T03:46:07,250 DEBUG [RS:1;c49966d8a95f:46637 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c49966d8a95f,46637,1733888766030' 2024-12-11T03:46:07,250 DEBUG [RS:1;c49966d8a95f:46637 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-11T03:46:07,250 DEBUG [RS:1;c49966d8a95f:46637 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-11T03:46:07,251 DEBUG [RS:1;c49966d8a95f:46637 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-11T03:46:07,251 INFO [RS:1;c49966d8a95f:46637 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-11T03:46:07,251 INFO [RS:1;c49966d8a95f:46637 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-11T03:46:07,268 WARN [c49966d8a95f:33041 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T03:46:07,354 INFO [RS:0;c49966d8a95f:40763 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T03:46:07,354 INFO [RS:1;c49966d8a95f:46637 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T03:46:07,355 INFO [RS:0;c49966d8a95f:40763 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c49966d8a95f%2C40763%2C1733888765890, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,40763,1733888765890, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/oldWALs, maxLogs=55 2024-12-11T03:46:07,355 INFO [RS:1;c49966d8a95f:46637 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c49966d8a95f%2C46637%2C1733888766030, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,46637,1733888766030, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/oldWALs, maxLogs=55 2024-12-11T03:46:07,378 INFO [RS:0;c49966d8a95f:40763 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,40763,1733888765890/c49966d8a95f%2C40763%2C1733888765890.1733888767359 2024-12-11T03:46:07,379 DEBUG [RS:0;c49966d8a95f:40763 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-11T03:46:07,379 INFO [RS:1;c49966d8a95f:46637 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,46637,1733888766030/c49966d8a95f%2C46637%2C1733888766030.1733888767359 2024-12-11T03:46:07,380 DEBUG [RS:1;c49966d8a95f:46637 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-11T03:46:07,520 DEBUG [c49966d8a95f:33041 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-11T03:46:07,536 DEBUG [c49966d8a95f:33041 {}] balancer.BalancerClusterState(204): Hosts are {c49966d8a95f=0} racks are {/default-rack=0} 2024-12-11T03:46:07,541 DEBUG [c49966d8a95f:33041 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-11T03:46:07,541 DEBUG [c49966d8a95f:33041 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-11T03:46:07,542 DEBUG [c49966d8a95f:33041 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-11T03:46:07,542 DEBUG [c49966d8a95f:33041 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-11T03:46:07,542 INFO [c49966d8a95f:33041 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-11T03:46:07,542 INFO [c49966d8a95f:33041 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-11T03:46:07,542 DEBUG [c49966d8a95f:33041 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-11T03:46:07,550 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c49966d8a95f,46637,1733888766030 2024-12-11T03:46:07,555 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c49966d8a95f,46637,1733888766030, state=OPENING 2024-12-11T03:46:07,561 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-11T03:46:07,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:07,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:07,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1019fa0d63e0002, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:07,564 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T03:46:07,564 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T03:46:07,564 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T03:46:07,566 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T03:46:07,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c49966d8a95f,46637,1733888766030}] 2024-12-11T03:46:07,740 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-11T03:46:07,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49011, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-11T03:46:07,752 INFO [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-11T03:46:07,752 INFO [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T03:46:07,753 INFO [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-11T03:46:07,754 INFO [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c49966d8a95f%2C46637%2C1733888766030.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,46637,1733888766030, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/oldWALs, maxLogs=55 2024-12-11T03:46:07,775 INFO [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,46637,1733888766030/c49966d8a95f%2C46637%2C1733888766030.meta.1733888767756.meta 2024-12-11T03:46:07,776 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-11T03:46:07,777 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-11T03:46:07,778 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-11T03:46:07,781 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-11T03:46:07,786 INFO [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-11T03:46:07,790 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-11T03:46:07,791 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T03:46:07,791 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-11T03:46:07,791 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-11T03:46:07,793 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-11T03:46:07,794 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-11T03:46:07,794 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:07,794 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T03:46:07,794 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-11T03:46:07,795 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-11T03:46:07,796 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:07,796 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T03:46:07,796 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-11T03:46:07,797 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-11T03:46:07,797 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:07,797 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T03:46:07,797 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-11T03:46:07,798 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-11T03:46:07,798 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:07,799 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T03:46:07,799 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-11T03:46:07,800 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740 2024-12-11T03:46:07,800 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740 2024-12-11T03:46:07,801 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-11T03:46:07,801 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-11T03:46:07,801 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T03:46:07,802 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-11T03:46:07,803 INFO [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70945581, jitterRate=0.0571715384721756}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T03:46:07,803 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-11T03:46:07,805 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733888767792Writing region info on filesystem at 1733888767792Initializing all the Stores at 1733888767792Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733888767792Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733888767793 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733888767793Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733888767793Cleaning up temporary data from old regions at 1733888767801 (+8 ms)Running coprocessor post-open hooks at 1733888767803 (+2 ms)Region opened successfully at 1733888767805 (+2 ms) 2024-12-11T03:46:07,811 INFO [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733888767732 2024-12-11T03:46:07,822 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-11T03:46:07,823 INFO [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-11T03:46:07,824 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c49966d8a95f,46637,1733888766030 2024-12-11T03:46:07,825 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c49966d8a95f,46637,1733888766030, state=OPEN 2024-12-11T03:46:07,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T03:46:07,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T03:46:07,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1019fa0d63e0002, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T03:46:07,828 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T03:46:07,828 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T03:46:07,828 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T03:46:07,828 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c49966d8a95f,46637,1733888766030 2024-12-11T03:46:07,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-11T03:46:07,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c49966d8a95f,46637,1733888766030 in 261 msec 2024-12-11T03:46:07,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-11T03:46:07,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 727 msec 2024-12-11T03:46:07,837 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T03:46:07,837 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-11T03:46:07,858 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T03:46:07,859 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c49966d8a95f,46637,1733888766030, seqNum=-1] 2024-12-11T03:46:07,880 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T03:46:07,882 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57161, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T03:46:07,904 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0000 sec 2024-12-11T03:46:07,905 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733888767904, completionTime=-1 2024-12-11T03:46:07,909 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-11T03:46:07,909 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T03:46:07,938 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T03:46:07,938 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733888827938 2024-12-11T03:46:07,938 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733888887938 2024-12-11T03:46:07,938 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-12-11T03:46:07,940 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-11T03:46:07,946 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,33041,1733888765137-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,946 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,33041,1733888765137-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,947 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,33041,1733888765137-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,948 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c49966d8a95f:33041, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,948 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,949 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:07,955 DEBUG [master/c49966d8a95f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-11T03:46:07,989 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.867sec 2024-12-11T03:46:07,991 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-11T03:46:07,992 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-11T03:46:07,993 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-11T03:46:07,994 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-11T03:46:07,994 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-11T03:46:07,997 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,33041,1733888765137-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T03:46:07,998 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,33041,1733888765137-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-11T03:46:07,999 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-11T03:46:08,000 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-11T03:46:08,000 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,33041,1733888765137-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:08,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ccab9e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T03:46:08,076 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-11T03:46:08,076 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-11T03:46:08,080 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c49966d8a95f,33041,-1 for getting cluster id 2024-12-11T03:46:08,082 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T03:46:08,090 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fd02e8ae-f0b8-4e5d-a2a1-b8508a87c1cf' 2024-12-11T03:46:08,092 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T03:46:08,093 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fd02e8ae-f0b8-4e5d-a2a1-b8508a87c1cf" 2024-12-11T03:46:08,095 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a8fa34d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T03:46:08,095 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c49966d8a95f,33041,-1] 2024-12-11T03:46:08,097 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T03:46:08,098 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T03:46:08,100 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T03:46:08,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e2571a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T03:46:08,103 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T03:46:08,109 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c49966d8a95f,46637,1733888766030, seqNum=-1] 2024-12-11T03:46:08,109 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T03:46:08,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T03:46:08,132 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c49966d8a95f,33041,1733888765137 2024-12-11T03:46:08,136 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T03:46:08,140 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c49966d8a95f,33041,1733888765137 2024-12-11T03:46:08,143 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@69048167 2024-12-11T03:46:08,144 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T03:46:08,146 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T03:46:08,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33041 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733888768132', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T03:46:08,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33041 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733888768132 2024-12-11T03:46:08,159 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733888768132 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T03:46:08,161 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33041 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733888768132" procId is: 4 2024-12-11T03:46:08,162 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:08,163 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733888768132 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T03:46:08,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T03:46:08,177 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733888768132-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 23a55b44de69fb8149fc5574f35c5e92, NAME => 'testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733888768132', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e 2024-12-11T03:46:08,183 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733888768132-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T03:46:08,184 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733888768132-pool-0 {}] regionserver.HRegion(1722): Closing 23a55b44de69fb8149fc5574f35c5e92, disabling compactions & flushes 2024-12-11T03:46:08,184 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733888768132-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:08,184 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733888768132-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:08,184 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733888768132-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. after waiting 0 ms 2024-12-11T03:46:08,184 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733888768132-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:08,184 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733888768132-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:08,184 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733888768132-pool-0 {}] regionserver.HRegion(1676): Region close journal for 23a55b44de69fb8149fc5574f35c5e92: Waiting for close lock at 1733888768184Disabling compacts and flushes for region at 1733888768184Disabling writes for close at 1733888768184Writing region close event to WAL at 1733888768184Closed at 1733888768184 2024-12-11T03:46:08,185 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733888768132 execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T03:46:08,190 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733888768185"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733888768185"}]},"ts":"1733888768185"} 2024-12-11T03:46:08,194 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T03:46:08,195 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733888768132 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T03:46:08,197 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733888768132","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733888768195"}]},"ts":"1733888768195"} 2024-12-11T03:46:08,201 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733888768132, state=ENABLING in hbase:meta 2024-12-11T03:46:08,202 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {c49966d8a95f=0} racks are {/default-rack=0} 2024-12-11T03:46:08,202 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-11T03:46:08,202 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-11T03:46:08,202 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-11T03:46:08,202 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-11T03:46:08,202 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-11T03:46:08,202 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-11T03:46:08,202 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-11T03:46:08,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733888768132, region=23a55b44de69fb8149fc5574f35c5e92, ASSIGN}] 2024-12-11T03:46:08,205 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733888768132, region=23a55b44de69fb8149fc5574f35c5e92, ASSIGN 2024-12-11T03:46:08,206 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733888768132, region=23a55b44de69fb8149fc5574f35c5e92, ASSIGN; state=OFFLINE, location=c49966d8a95f,40763,1733888765890; forceNewPlan=false, retain=false 2024-12-11T03:46:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T03:46:08,358 INFO [c49966d8a95f:33041 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-11T03:46:08,359 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=23a55b44de69fb8149fc5574f35c5e92, regionState=OPENING, regionLocation=c49966d8a95f,40763,1733888765890 2024-12-11T03:46:08,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733888768132, region=23a55b44de69fb8149fc5574f35c5e92, ASSIGN because future has completed 2024-12-11T03:46:08,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a55b44de69fb8149fc5574f35c5e92, server=c49966d8a95f,40763,1733888765890}] 2024-12-11T03:46:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T03:46:08,517 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-11T03:46:08,519 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37027, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-11T03:46:08,521 INFO [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:08,522 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 23a55b44de69fb8149fc5574f35c5e92, NAME => 'testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92.', STARTKEY => '', ENDKEY => ''} 2024-12-11T03:46:08,522 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733888768132 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:08,522 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T03:46:08,522 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:08,522 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:08,523 INFO [StoreOpener-23a55b44de69fb8149fc5574f35c5e92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:08,528 INFO [StoreOpener-23a55b44de69fb8149fc5574f35c5e92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23a55b44de69fb8149fc5574f35c5e92 columnFamilyName family 2024-12-11T03:46:08,528 DEBUG [StoreOpener-23a55b44de69fb8149fc5574f35c5e92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:08,529 INFO [StoreOpener-23a55b44de69fb8149fc5574f35c5e92-1 {}] regionserver.HStore(327): Store=23a55b44de69fb8149fc5574f35c5e92/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T03:46:08,529 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:08,530 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:08,530 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:08,530 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:08,530 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:08,531 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:08,538 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T03:46:08,538 INFO [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 23a55b44de69fb8149fc5574f35c5e92; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66112578, jitterRate=-0.014845818281173706}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T03:46:08,539 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:08,539 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 23a55b44de69fb8149fc5574f35c5e92: Running coprocessor pre-open hook at 1733888768523Writing region info on filesystem at 1733888768523Initializing all the Stores at 1733888768523Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733888768523Cleaning up temporary data from old regions at 1733888768530 (+7 ms)Running coprocessor post-open hooks at 1733888768539 (+9 ms)Region opened successfully at 1733888768539 2024-12-11T03:46:08,541 INFO [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92., pid=6, masterSystemTime=1733888768516 2024-12-11T03:46:08,543 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:08,544 INFO [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:08,544 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=23a55b44de69fb8149fc5574f35c5e92, regionState=OPEN, openSeqNum=2, regionLocation=c49966d8a95f,40763,1733888765890 2024-12-11T03:46:08,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a55b44de69fb8149fc5574f35c5e92, server=c49966d8a95f,40763,1733888765890 because future has completed 2024-12-11T03:46:08,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-11T03:46:08,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 23a55b44de69fb8149fc5574f35c5e92, server=c49966d8a95f,40763,1733888765890 in 185 msec 2024-12-11T03:46:08,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-11T03:46:08,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733888768132, region=23a55b44de69fb8149fc5574f35c5e92, ASSIGN in 347 msec 2024-12-11T03:46:08,554 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733888768132 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T03:46:08,554 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733888768132","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733888768554"}]},"ts":"1733888768554"} 2024-12-11T03:46:08,557 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733888768132, state=ENABLED in hbase:meta 2024-12-11T03:46:08,558 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733888768132 execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T03:46:08,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733888768132 in 404 msec 2024-12-11T03:46:08,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T03:46:08,798 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733888768132 get assigned. Timeout = 60000ms 2024-12-11T03:46:08,798 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733888768132 completed 2024-12-11T03:46:08,802 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T03:46:08,807 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733888768132 assigned to meta. Checking AM states. 2024-12-11T03:46:08,808 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T03:46:08,808 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733888768132 assigned. 2024-12-11T03:46:08,817 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733888768132', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92., hostname=c49966d8a95f,40763,1733888765890, seqNum=2] 2024-12-11T03:46:08,818 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T03:46:08,820 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T03:46:08,831 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 23a55b44de69fb8149fc5574f35c5e92 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-11T03:46:08,861 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fa2846e82bdb424897a7247e3887e1e8 2024-12-11T03:46:08,899 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92/.tmp/family/fa2846e82bdb424897a7247e3887e1e8 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-11T03:46:08,904 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92/.tmp/family/fa2846e82bdb424897a7247e3887e1e8 2024-12-11T03:46:08,940 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92/.tmp/family/fa2846e82bdb424897a7247e3887e1e8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92/family/fa2846e82bdb424897a7247e3887e1e8 2024-12-11T03:46:08,942 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92/family/fa2846e82bdb424897a7247e3887e1e8, delay=933 ms 2024-12-11T03:46:08,944 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92/family/fa2846e82bdb424897a7247e3887e1e8, entries=2, sequenceid=6, filesize=4.9 K 2024-12-11T03:46:08,951 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 23a55b44de69fb8149fc5574f35c5e92 in 119ms, sequenceid=6, compaction requested=false 2024-12-11T03:46:08,953 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733888768132' 2024-12-11T03:46:08,955 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 23a55b44de69fb8149fc5574f35c5e92: 2024-12-11T03:46:09,878 DEBUG [hfile-prefetch-1733888768943 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fa2846e82bdb424897a7247e3887e1e8_0. 2024-12-11T03:46:09,882 DEBUG [hfile-prefetch-1733888768943 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fa2846e82bdb424897a7247e3887e1e8_117. 2024-12-11T03:46:09,883 DEBUG [hfile-prefetch-1733888768943 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92/family/fa2846e82bdb424897a7247e3887e1e8, with total blocks 2, and data blocks 1 2024-12-11T03:46:09,883 DEBUG [hfile-prefetch-1733888768943 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-11T03:46:09,883 DEBUG [hfile-prefetch-1733888768943 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92/family/fa2846e82bdb424897a7247e3887e1e8, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-11T03:46:09,883 DEBUG [hfile-prefetch-1733888768943 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92/family/fa2846e82bdb424897a7247e3887e1e8. Will sleep 100ms and try the verification again. 2024-12-11T03:46:09,949 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-11T03:46:09,949 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-11T03:46:09,949 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T03:46:09,949 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-11T03:46:09,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33041 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=23a55b44de69fb8149fc5574f35c5e92, source=c49966d8a95f,40763,1733888765890, destination=c49966d8a95f,46637,1733888766030, warming up region on c49966d8a95f,46637,1733888766030 2024-12-11T03:46:09,968 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-11T03:46:09,968 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733888769949 2024-12-11T03:46:09,968 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-11T03:46:09,970 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33041 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=23a55b44de69fb8149fc5574f35c5e92, source=c49966d8a95f,40763,1733888765890, destination=c49966d8a95f,46637,1733888766030, running balancer 2024-12-11T03:46:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33041 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733888768132, region=23a55b44de69fb8149fc5574f35c5e92, REOPEN/MOVE 2024-12-11T03:46:09,971 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733888768132, region=23a55b44de69fb8149fc5574f35c5e92, REOPEN/MOVE 2024-12-11T03:46:09,971 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46637 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46637 {}] regionserver.HRegion(7855): Warmup {ENCODED => 23a55b44de69fb8149fc5574f35c5e92, NAME => 'testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92.', STARTKEY => '', ENDKEY => ''} 2024-12-11T03:46:09,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46637 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T03:46:09,973 INFO [StoreOpener-23a55b44de69fb8149fc5574f35c5e92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:09,973 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=23a55b44de69fb8149fc5574f35c5e92, regionState=CLOSING, regionLocation=c49966d8a95f,40763,1733888765890 2024-12-11T03:46:09,974 INFO [StoreOpener-23a55b44de69fb8149fc5574f35c5e92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23a55b44de69fb8149fc5574f35c5e92 columnFamilyName family 2024-12-11T03:46:09,974 DEBUG [StoreOpener-23a55b44de69fb8149fc5574f35c5e92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:09,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733888768132, region=23a55b44de69fb8149fc5574f35c5e92, REOPEN/MOVE because future has completed 2024-12-11T03:46:09,976 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-11T03:46:09,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 23a55b44de69fb8149fc5574f35c5e92, server=c49966d8a95f,40763,1733888765890}] 2024-12-11T03:46:09,977 DEBUG [StoreOpener-23a55b44de69fb8149fc5574f35c5e92-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92/family/fa2846e82bdb424897a7247e3887e1e8 2024-12-11T03:46:09,977 INFO [StoreOpener-23a55b44de69fb8149fc5574f35c5e92-1 {}] regionserver.HStore(327): Store=23a55b44de69fb8149fc5574f35c5e92/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T03:46:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46637 {}] regionserver.HRegion(1722): Closing 23a55b44de69fb8149fc5574f35c5e92, disabling compactions & flushes 2024-12-11T03:46:09,978 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46637 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46637 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46637 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. after waiting 0 ms 2024-12-11T03:46:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46637 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:09,979 DEBUG [StoreFileCloser-23a55b44de69fb8149fc5574f35c5e92-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92/family/fa2846e82bdb424897a7247e3887e1e8 2024-12-11T03:46:09,979 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46637 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:09,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46637 {}] regionserver.HRegion(1676): Region close journal for 23a55b44de69fb8149fc5574f35c5e92: Waiting for close lock at 1733888769977Disabling compacts and flushes for region at 1733888769977Disabling writes for close at 1733888769978 (+1 ms)Writing region close event to WAL at 1733888769979 (+1 ms)Closed at 1733888769979 2024-12-11T03:46:09,981 WARN [hfile-prefetch-1733888768943 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92/family/fa2846e82bdb424897a7247e3887e1e8, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-11T03:46:09,982 DEBUG [hfile-prefetch-1733888768943 {}] hfile.PrefetchExecutor(121): Prefetch completed for fa2846e82bdb424897a7247e3887e1e8 2024-12-11T03:46:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33041 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-11T03:46:10,134 INFO [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:10,134 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-11T03:46:10,135 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 23a55b44de69fb8149fc5574f35c5e92, disabling compactions & flushes 2024-12-11T03:46:10,135 INFO [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:10,136 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:10,136 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. after waiting 0 ms 2024-12-11T03:46:10,136 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:10,137 DEBUG [StoreFileCloser-23a55b44de69fb8149fc5574f35c5e92-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file fa2846e82bdb424897a7247e3887e1e8, starting offset: 0, end offset: 9223372036854775807 2024-12-11T03:46:10,139 DEBUG [StoreFileCloser-23a55b44de69fb8149fc5574f35c5e92-family-1 {}] bucket.BucketCache(856): removed key fa2846e82bdb424897a7247e3887e1e8_0 from back map with offset lock 2094080 in the evict process 2024-12-11T03:46:10,144 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-11T03:46:10,145 INFO [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:10,145 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 23a55b44de69fb8149fc5574f35c5e92: Waiting for close lock at 1733888770135Running coprocessor pre-close hooks at 1733888770135Disabling compacts and flushes for region at 1733888770135Disabling writes for close at 1733888770136 (+1 ms)Writing region close event to WAL at 1733888770139 (+3 ms)Running coprocessor post-close hooks at 1733888770144 (+5 ms)Closed at 1733888770145 (+1 ms) 2024-12-11T03:46:10,146 INFO [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 23a55b44de69fb8149fc5574f35c5e92 move to c49966d8a95f,46637,1733888766030 record at close sequenceid=6 2024-12-11T03:46:10,149 INFO [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:10,150 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=23a55b44de69fb8149fc5574f35c5e92, regionState=CLOSED 2024-12-11T03:46:10,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 23a55b44de69fb8149fc5574f35c5e92, server=c49966d8a95f,40763,1733888765890 because future has completed 2024-12-11T03:46:10,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-11T03:46:10,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 23a55b44de69fb8149fc5574f35c5e92, server=c49966d8a95f,40763,1733888765890 in 177 msec 2024-12-11T03:46:10,156 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733888768132, region=23a55b44de69fb8149fc5574f35c5e92, REOPEN/MOVE; state=CLOSED, location=c49966d8a95f,46637,1733888766030; forceNewPlan=false, retain=false 2024-12-11T03:46:10,169 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-11T03:46:10,169 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-11T03:46:10,169 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T03:46:10,169 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-11T03:46:10,172 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-11T03:46:10,172 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733888770169 2024-12-11T03:46:10,172 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-11T03:46:10,307 INFO [c49966d8a95f:33041 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-11T03:46:10,307 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=23a55b44de69fb8149fc5574f35c5e92, regionState=OPENING, regionLocation=c49966d8a95f,46637,1733888766030 2024-12-11T03:46:10,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733888768132, region=23a55b44de69fb8149fc5574f35c5e92, REOPEN/MOVE because future has completed 2024-12-11T03:46:10,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a55b44de69fb8149fc5574f35c5e92, server=c49966d8a95f,46637,1733888766030}] 2024-12-11T03:46:10,465 INFO [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:10,465 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 23a55b44de69fb8149fc5574f35c5e92, NAME => 'testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92.', STARTKEY => '', ENDKEY => ''} 2024-12-11T03:46:10,466 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733888768132 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:10,466 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T03:46:10,466 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:10,466 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:10,467 INFO [StoreOpener-23a55b44de69fb8149fc5574f35c5e92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:10,468 INFO [StoreOpener-23a55b44de69fb8149fc5574f35c5e92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23a55b44de69fb8149fc5574f35c5e92 columnFamilyName family 2024-12-11T03:46:10,468 DEBUG [StoreOpener-23a55b44de69fb8149fc5574f35c5e92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:10,470 DEBUG [StoreFileOpener-23a55b44de69fb8149fc5574f35c5e92-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92/family/fa2846e82bdb424897a7247e3887e1e8, delay=945 ms 2024-12-11T03:46:10,471 DEBUG [StoreOpener-23a55b44de69fb8149fc5574f35c5e92-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92/family/fa2846e82bdb424897a7247e3887e1e8 2024-12-11T03:46:10,471 INFO [StoreOpener-23a55b44de69fb8149fc5574f35c5e92-1 {}] regionserver.HStore(327): Store=23a55b44de69fb8149fc5574f35c5e92/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T03:46:10,472 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:10,472 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:10,473 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:10,473 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:10,473 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:10,478 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:10,478 INFO [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 23a55b44de69fb8149fc5574f35c5e92; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68846955, jitterRate=0.025899574160575867}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T03:46:10,478 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:10,480 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 23a55b44de69fb8149fc5574f35c5e92: Running coprocessor pre-open hook at 1733888770466Writing region info on filesystem at 1733888770466Initializing all the Stores at 1733888770467 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733888770467Cleaning up temporary data from old regions at 1733888770473 (+6 ms)Running coprocessor post-open hooks at 1733888770479 (+6 ms)Region opened successfully at 1733888770480 (+1 ms) 2024-12-11T03:46:10,481 INFO [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92., pid=9, masterSystemTime=1733888770462 2024-12-11T03:46:10,484 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=23a55b44de69fb8149fc5574f35c5e92, regionState=OPEN, openSeqNum=10, regionLocation=c49966d8a95f,46637,1733888766030 2024-12-11T03:46:10,485 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:10,485 INFO [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:10,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a55b44de69fb8149fc5574f35c5e92, server=c49966d8a95f,46637,1733888766030 because future has completed 2024-12-11T03:46:10,490 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-11T03:46:10,490 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 23a55b44de69fb8149fc5574f35c5e92, server=c49966d8a95f,46637,1733888766030 in 177 msec 2024-12-11T03:46:10,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733888768132, region=23a55b44de69fb8149fc5574f35c5e92, REOPEN/MOVE in 520 msec 2024-12-11T03:46:10,524 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-11T03:46:10,524 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T03:46:10,524 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T03:46:10,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T03:46:10,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T03:46:10,530 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T03:46:10,530 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-11T03:46:10,530 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=873394183, stopped=false 2024-12-11T03:46:10,531 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c49966d8a95f,33041,1733888765137 2024-12-11T03:46:10,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-11T03:46:10,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1019fa0d63e0002, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-11T03:46:10,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-11T03:46:10,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:10,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1019fa0d63e0002, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:10,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:10,533 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T03:46:10,533 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T03:46:10,534 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T03:46:10,534 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T03:46:10,534 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T03:46:10,534 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T03:46:10,534 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c49966d8a95f,40763,1733888765890' ***** 2024-12-11T03:46:10,534 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-11T03:46:10,534 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46637-0x1019fa0d63e0002, quorum=127.0.0.1:57764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T03:46:10,534 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c49966d8a95f,46637,1733888766030' ***** 2024-12-11T03:46:10,534 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-11T03:46:10,535 INFO [RS:0;c49966d8a95f:40763 {}] bucket.BucketCache(1738): Disabling cache 2024-12-11T03:46:10,535 INFO [RS:1;c49966d8a95f:46637 {}] bucket.BucketCache(1738): Disabling cache 2024-12-11T03:46:10,535 INFO [RS:0;c49966d8a95f:40763 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/bucket.persistence 2024-12-11T03:46:10,535 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-11T03:46:10,535 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-11T03:46:10,535 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-11T03:46:10,535 INFO [RS:1;c49966d8a95f:46637 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/bucket.persistence 2024-12-11T03:46:10,535 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-11T03:46:10,535 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-11T03:46:10,535 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-11T03:46:10,536 INFO [RS:0;c49966d8a95f:40763 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-11T03:46:10,536 INFO [RS:1;c49966d8a95f:46637 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-11T03:46:10,536 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-11T03:46:10,536 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-11T03:46:10,546 DEBUG [RS:0;c49966d8a95f:40763 {}] bucket.BucketCache(1487): Thread RS:0;c49966d8a95f:40763 started persisting bucket cache to file 2024-12-11T03:46:10,546 DEBUG [RS:1;c49966d8a95f:46637 {}] bucket.BucketCache(1487): Thread RS:1;c49966d8a95f:46637 started persisting bucket cache to file 2024-12-11T03:46:10,546 DEBUG [RS:0;c49966d8a95f:40763 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T03:46:10,546 DEBUG [RS:1;c49966d8a95f:46637 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T03:46:10,546 DEBUG [RS:0;c49966d8a95f:40763 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-11T03:46:10,546 DEBUG [RS:1;c49966d8a95f:46637 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-11T03:46:10,550 DEBUG [RS:1;c49966d8a95f:46637 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-11T03:46:10,550 DEBUG [RS:1;c49966d8a95f:46637 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733888770546 2024-12-11T03:46:10,550 DEBUG [RS:1;c49966d8a95f:46637 {}] bucket.BucketCache(1509): Thread RS:1;c49966d8a95f:46637 finished persisting bucket cache to file, renaming 2024-12-11T03:46:10,550 DEBUG [RS:0;c49966d8a95f:40763 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-11T03:46:10,550 DEBUG [RS:0;c49966d8a95f:40763 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733888770546 2024-12-11T03:46:10,550 DEBUG [RS:0;c49966d8a95f:40763 {}] bucket.BucketCache(1509): Thread RS:0;c49966d8a95f:40763 finished persisting bucket cache to file, renaming 2024-12-11T03:46:10,550 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.HeapMemoryManager(224): Stopping 2024-12-11T03:46:10,550 WARN [RS:0;c49966d8a95f:40763 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-11T03:46:10,550 INFO [RS:0;c49966d8a95f:40763 {}] regionserver.HeapMemoryManager(224): Stopping 2024-12-11T03:46:10,551 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-11T03:46:10,551 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-11T03:46:10,551 INFO [RS:1;c49966d8a95f:46637 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-11T03:46:10,551 INFO [RS:0;c49966d8a95f:40763 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-11T03:46:10,551 INFO [RS:0;c49966d8a95f:40763 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-11T03:46:10,551 INFO [RS:1;c49966d8a95f:46637 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-11T03:46:10,551 INFO [RS:0;c49966d8a95f:40763 {}] regionserver.HRegionServer(959): stopping server c49966d8a95f,40763,1733888765890 2024-12-11T03:46:10,551 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.HRegionServer(3091): Received CLOSE for 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:10,551 INFO [RS:0;c49966d8a95f:40763 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T03:46:10,551 INFO [RS:0;c49966d8a95f:40763 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c49966d8a95f:40763. 2024-12-11T03:46:10,551 DEBUG [RS:0;c49966d8a95f:40763 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T03:46:10,551 DEBUG [RS:0;c49966d8a95f:40763 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T03:46:10,551 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.HRegionServer(959): stopping server c49966d8a95f,46637,1733888766030 2024-12-11T03:46:10,551 INFO [RS:0;c49966d8a95f:40763 {}] regionserver.HRegionServer(976): stopping server c49966d8a95f,40763,1733888765890; all regions closed. 2024-12-11T03:46:10,551 INFO [RS:1;c49966d8a95f:46637 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T03:46:10,552 INFO [RS:1;c49966d8a95f:46637 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;c49966d8a95f:46637. 2024-12-11T03:46:10,552 DEBUG [RS:1;c49966d8a95f:46637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T03:46:10,552 DEBUG [RS:1;c49966d8a95f:46637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T03:46:10,552 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-11T03:46:10,552 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-11T03:46:10,552 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-11T03:46:10,552 INFO [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-11T03:46:10,552 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-11T03:46:10,552 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 23a55b44de69fb8149fc5574f35c5e92, disabling compactions & flushes 2024-12-11T03:46:10,552 INFO [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:10,552 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:10,553 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-11T03:46:10,553 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. after waiting 1 ms 2024-12-11T03:46:10,553 DEBUG [RS:1;c49966d8a95f:46637 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 23a55b44de69fb8149fc5574f35c5e92=testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92.} 2024-12-11T03:46:10,553 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:10,553 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-11T03:46:10,553 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-11T03:46:10,553 DEBUG [RS:1;c49966d8a95f:46637 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 23a55b44de69fb8149fc5574f35c5e92 2024-12-11T03:46:10,553 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-11T03:46:10,553 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-11T03:46:10,553 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-11T03:46:10,553 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-11T03:46:10,553 DEBUG [StoreFileCloser-23a55b44de69fb8149fc5574f35c5e92-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92/family/fa2846e82bdb424897a7247e3887e1e8 2024-12-11T03:46:10,554 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-11T03:46:10,558 DEBUG [RS:0;c49966d8a95f:40763 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/oldWALs 2024-12-11T03:46:10,558 INFO [RS:0;c49966d8a95f:40763 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c49966d8a95f%2C40763%2C1733888765890:(num 1733888767359) 2024-12-11T03:46:10,558 DEBUG [RS:0;c49966d8a95f:40763 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T03:46:10,558 INFO [RS:0;c49966d8a95f:40763 {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T03:46:10,558 INFO [RS:0;c49966d8a95f:40763 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T03:46:10,558 INFO [RS:0;c49966d8a95f:40763 {}] hbase.ChoreService(370): Chore service for: regionserver/c49966d8a95f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-11T03:46:10,559 INFO [RS:0;c49966d8a95f:40763 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-11T03:46:10,559 INFO [RS:0;c49966d8a95f:40763 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-11T03:46:10,559 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnRegionMove1733888768132/23a55b44de69fb8149fc5574f35c5e92/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-11T03:46:10,559 INFO [RS:0;c49966d8a95f:40763 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-11T03:46:10,559 INFO [regionserver/c49966d8a95f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T03:46:10,559 INFO [RS:0;c49966d8a95f:40763 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T03:46:10,559 INFO [RS:0;c49966d8a95f:40763 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40763 2024-12-11T03:46:10,559 INFO [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:10,560 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 23a55b44de69fb8149fc5574f35c5e92: Waiting for close lock at 1733888770552Running coprocessor pre-close hooks at 1733888770552Disabling compacts and flushes for region at 1733888770552Disabling writes for close at 1733888770553 (+1 ms)Writing region close event to WAL at 1733888770554 (+1 ms)Running coprocessor post-close hooks at 1733888770559 (+5 ms)Closed at 1733888770559 2024-12-11T03:46:10,560 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92. 2024-12-11T03:46:10,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c49966d8a95f,40763,1733888765890 2024-12-11T03:46:10,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T03:46:10,564 INFO [RS:0;c49966d8a95f:40763 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T03:46:10,567 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5cbd675d1f934ae285cc6308eb8a2d2c 2024-12-11T03:46:10,569 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c49966d8a95f,40763,1733888765890] 2024-12-11T03:46:10,570 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c49966d8a95f,40763,1733888765890 already deleted, retry=false 2024-12-11T03:46:10,570 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c49966d8a95f,40763,1733888765890 expired; onlineServers=1 2024-12-11T03:46:10,584 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/.tmp/info/5cbd675d1f934ae285cc6308eb8a2d2c is 205, key is testBlockEvictionOnRegionMove1733888768132,,1733888768148.23a55b44de69fb8149fc5574f35c5e92./info:regioninfo/1733888770484/Put/seqid=0 2024-12-11T03:46:10,585 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/.tmp/info/5cbd675d1f934ae285cc6308eb8a2d2c 2024-12-11T03:46:10,594 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d15776cec2014038b613ed358e717a74 2024-12-11T03:46:10,600 INFO [regionserver/c49966d8a95f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T03:46:10,600 INFO [regionserver/c49966d8a95f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T03:46:10,608 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/.tmp/ns/d15776cec2014038b613ed358e717a74 is 43, key is default/ns:d/1733888767887/Put/seqid=0 2024-12-11T03:46:10,609 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/.tmp/ns/d15776cec2014038b613ed358e717a74 2024-12-11T03:46:10,618 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5064da012de04223b261c3566e1d2006 2024-12-11T03:46:10,632 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/.tmp/table/5064da012de04223b261c3566e1d2006 is 78, key is testBlockEvictionOnRegionMove1733888768132/table:state/1733888768554/Put/seqid=0 2024-12-11T03:46:10,633 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/.tmp/table/5064da012de04223b261c3566e1d2006 2024-12-11T03:46:10,636 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/.tmp/info/5cbd675d1f934ae285cc6308eb8a2d2c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/info/5cbd675d1f934ae285cc6308eb8a2d2c 2024-12-11T03:46:10,637 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/info/5cbd675d1f934ae285cc6308eb8a2d2c, delay=965 ms 2024-12-11T03:46:10,638 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/info/5cbd675d1f934ae285cc6308eb8a2d2c, entries=15, sequenceid=15, filesize=7.6 K 2024-12-11T03:46:10,639 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/.tmp/ns/d15776cec2014038b613ed358e717a74 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/ns/d15776cec2014038b613ed358e717a74 2024-12-11T03:46:10,640 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/ns/d15776cec2014038b613ed358e717a74, delay=989 ms 2024-12-11T03:46:10,641 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/ns/d15776cec2014038b613ed358e717a74, entries=2, sequenceid=15, filesize=5.0 K 2024-12-11T03:46:10,642 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/.tmp/table/5064da012de04223b261c3566e1d2006 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/table/5064da012de04223b261c3566e1d2006 2024-12-11T03:46:10,643 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/table/5064da012de04223b261c3566e1d2006, delay=982 ms 2024-12-11T03:46:10,644 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/table/5064da012de04223b261c3566e1d2006, entries=2, sequenceid=15, filesize=5.3 K 2024-12-11T03:46:10,645 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 92ms, sequenceid=15, compaction requested=false 2024-12-11T03:46:10,645 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-11T03:46:10,646 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/info/5cbd675d1f934ae285cc6308eb8a2d2c 2024-12-11T03:46:10,647 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/ns/d15776cec2014038b613ed358e717a74 2024-12-11T03:46:10,647 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/table/5064da012de04223b261c3566e1d2006 2024-12-11T03:46:10,652 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-11T03:46:10,653 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-11T03:46:10,653 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-11T03:46:10,653 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733888770553Running coprocessor pre-close hooks at 1733888770553Disabling compacts and flushes for region at 1733888770553Disabling writes for close at 1733888770553Obtaining lock to block concurrent updates at 1733888770554 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733888770554Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733888770554Flushing stores of hbase:meta,,1.1588230740 at 1733888770555 (+1 ms)Flushing 1588230740/info: creating writer at 1733888770555Flushing 1588230740/info: appending metadata at 1733888770583 (+28 ms)Flushing 1588230740/info: closing flushed file at 1733888770583Flushing 1588230740/ns: creating writer at 1733888770587 (+4 ms)Flushing 1588230740/ns: appending metadata at 1733888770607 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1733888770607Flushing 1588230740/table: creating writer at 1733888770611 (+4 ms)Flushing 1588230740/table: appending metadata at 1733888770632 (+21 ms)Flushing 1588230740/table: closing flushed file at 1733888770632Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bbd0645: reopening flushed file at 1733888770635 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f6b858d: reopening flushed file at 1733888770638 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@173cfb5f: reopening flushed file at 1733888770641 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 92ms, sequenceid=15, compaction requested=false at 1733888770645 (+4 ms)Writing region close event to WAL at 1733888770648 (+3 ms)Running coprocessor post-close hooks at 1733888770652 (+4 ms)Closed at 1733888770653 (+1 ms) 2024-12-11T03:46:10,653 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-11T03:46:10,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T03:46:10,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40763-0x1019fa0d63e0001, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T03:46:10,669 INFO [RS:0;c49966d8a95f:40763 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-11T03:46:10,670 INFO [RS:0;c49966d8a95f:40763 {}] regionserver.HRegionServer(1031): Exiting; stopping=c49966d8a95f,40763,1733888765890; zookeeper connection closed. 2024-12-11T03:46:10,670 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@48698d04 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@48698d04 2024-12-11T03:46:10,753 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.HRegionServer(976): stopping server c49966d8a95f,46637,1733888766030; all regions closed. 2024-12-11T03:46:10,756 DEBUG [RS:1;c49966d8a95f:46637 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/oldWALs 2024-12-11T03:46:10,756 INFO [RS:1;c49966d8a95f:46637 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c49966d8a95f%2C46637%2C1733888766030.meta:.meta(num 1733888767756) 2024-12-11T03:46:10,758 DEBUG [RS:1;c49966d8a95f:46637 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/oldWALs 2024-12-11T03:46:10,758 INFO [RS:1;c49966d8a95f:46637 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c49966d8a95f%2C46637%2C1733888766030:(num 1733888767359) 2024-12-11T03:46:10,758 DEBUG [RS:1;c49966d8a95f:46637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T03:46:10,758 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T03:46:10,758 INFO [RS:1;c49966d8a95f:46637 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T03:46:10,759 INFO [RS:1;c49966d8a95f:46637 {}] hbase.ChoreService(370): Chore service for: regionserver/c49966d8a95f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-11T03:46:10,759 INFO [RS:1;c49966d8a95f:46637 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T03:46:10,759 INFO [regionserver/c49966d8a95f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T03:46:10,759 INFO [RS:1;c49966d8a95f:46637 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46637 2024-12-11T03:46:10,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1019fa0d63e0002, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c49966d8a95f,46637,1733888766030 2024-12-11T03:46:10,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T03:46:10,762 INFO [RS:1;c49966d8a95f:46637 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T03:46:10,763 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c49966d8a95f,46637,1733888766030] 2024-12-11T03:46:10,766 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c49966d8a95f,46637,1733888766030 already deleted, retry=false 2024-12-11T03:46:10,766 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c49966d8a95f,46637,1733888766030 expired; onlineServers=0 2024-12-11T03:46:10,767 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c49966d8a95f,33041,1733888765137' ***** 2024-12-11T03:46:10,767 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-11T03:46:10,767 INFO [M:0;c49966d8a95f:33041 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T03:46:10,767 INFO [M:0;c49966d8a95f:33041 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T03:46:10,767 DEBUG [M:0;c49966d8a95f:33041 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-11T03:46:10,767 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-11T03:46:10,767 DEBUG [M:0;c49966d8a95f:33041 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-11T03:46:10,767 DEBUG [master/c49966d8a95f:0:becomeActiveMaster-HFileCleaner.small.0-1733888767026 {}] cleaner.HFileCleaner(306): Exit Thread[master/c49966d8a95f:0:becomeActiveMaster-HFileCleaner.small.0-1733888767026,5,FailOnTimeoutGroup] 2024-12-11T03:46:10,767 DEBUG [master/c49966d8a95f:0:becomeActiveMaster-HFileCleaner.large.0-1733888767025 {}] cleaner.HFileCleaner(306): Exit Thread[master/c49966d8a95f:0:becomeActiveMaster-HFileCleaner.large.0-1733888767025,5,FailOnTimeoutGroup] 2024-12-11T03:46:10,767 INFO [M:0;c49966d8a95f:33041 {}] hbase.ChoreService(370): Chore service for: master/c49966d8a95f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-11T03:46:10,768 INFO [M:0;c49966d8a95f:33041 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T03:46:10,768 DEBUG [M:0;c49966d8a95f:33041 {}] master.HMaster(1795): Stopping service threads 2024-12-11T03:46:10,768 INFO [M:0;c49966d8a95f:33041 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-11T03:46:10,768 INFO [M:0;c49966d8a95f:33041 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T03:46:10,768 INFO [M:0;c49966d8a95f:33041 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T03:46:10,768 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-11T03:46:10,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-11T03:46:10,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:10,771 DEBUG [M:0;c49966d8a95f:33041 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-12-11T03:46:10,771 DEBUG [M:0;c49966d8a95f:33041 {}] master.ActiveMasterManager(353): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-12-11T03:46:10,771 INFO [M:0;c49966d8a95f:33041 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/.lastflushedseqids 2024-12-11T03:46:10,784 INFO [M:0;c49966d8a95f:33041 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T03:46:10,784 INFO [M:0;c49966d8a95f:33041 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-11T03:46:10,784 DEBUG [M:0;c49966d8a95f:33041 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-11T03:46:10,784 INFO [M:0;c49966d8a95f:33041 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T03:46:10,784 DEBUG [M:0;c49966d8a95f:33041 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T03:46:10,785 DEBUG [M:0;c49966d8a95f:33041 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-11T03:46:10,785 DEBUG [M:0;c49966d8a95f:33041 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T03:46:10,785 INFO [M:0;c49966d8a95f:33041 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-12-11T03:46:10,795 DEBUG [M:0;c49966d8a95f:33041 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b5fa5001633341059e40db0fd1392919 2024-12-11T03:46:10,809 DEBUG [M:0;c49966d8a95f:33041 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b5fa5001633341059e40db0fd1392919 is 82, key is hbase:meta,,1/info:regioninfo/1733888767823/Put/seqid=0 2024-12-11T03:46:10,810 INFO [M:0;c49966d8a95f:33041 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b5fa5001633341059e40db0fd1392919 2024-12-11T03:46:10,818 DEBUG [M:0;c49966d8a95f:33041 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a892d60365614fda9bfda4c89746a703 2024-12-11T03:46:10,833 DEBUG [M:0;c49966d8a95f:33041 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a892d60365614fda9bfda4c89746a703 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733888768559/Put/seqid=0 2024-12-11T03:46:10,834 INFO [M:0;c49966d8a95f:33041 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a892d60365614fda9bfda4c89746a703 2024-12-11T03:46:10,843 DEBUG [M:0;c49966d8a95f:33041 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 88ef9886353946f2a3ea0f9293f9a354 2024-12-11T03:46:10,859 DEBUG [M:0;c49966d8a95f:33041 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/88ef9886353946f2a3ea0f9293f9a354 is 69, key is c49966d8a95f,40763,1733888765890/rs:state/1733888767110/Put/seqid=0 2024-12-11T03:46:10,860 INFO [M:0;c49966d8a95f:33041 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/88ef9886353946f2a3ea0f9293f9a354 2024-12-11T03:46:10,862 DEBUG [M:0;c49966d8a95f:33041 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b5fa5001633341059e40db0fd1392919 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b5fa5001633341059e40db0fd1392919 2024-12-11T03:46:10,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1019fa0d63e0002, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T03:46:10,864 INFO [M:0;c49966d8a95f:33041 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b5fa5001633341059e40db0fd1392919, entries=8, sequenceid=84, filesize=5.5 K 2024-12-11T03:46:10,864 INFO [RS:1;c49966d8a95f:46637 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-11T03:46:10,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1019fa0d63e0002, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T03:46:10,864 INFO [RS:1;c49966d8a95f:46637 {}] regionserver.HRegionServer(1031): Exiting; stopping=c49966d8a95f,46637,1733888766030; zookeeper connection closed. 2024-12-11T03:46:10,865 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@733c8d55 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@733c8d55 2024-12-11T03:46:10,865 DEBUG [M:0;c49966d8a95f:33041 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a892d60365614fda9bfda4c89746a703 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a892d60365614fda9bfda4c89746a703 2024-12-11T03:46:10,865 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-11T03:46:10,867 INFO [M:0;c49966d8a95f:33041 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a892d60365614fda9bfda4c89746a703, entries=9, sequenceid=84, filesize=6.5 K 2024-12-11T03:46:10,868 DEBUG [M:0;c49966d8a95f:33041 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/88ef9886353946f2a3ea0f9293f9a354 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/88ef9886353946f2a3ea0f9293f9a354 2024-12-11T03:46:10,870 INFO [M:0;c49966d8a95f:33041 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/88ef9886353946f2a3ea0f9293f9a354, entries=2, sequenceid=84, filesize=5.1 K 2024-12-11T03:46:10,871 INFO [M:0;c49966d8a95f:33041 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 86ms, sequenceid=84, compaction requested=false 2024-12-11T03:46:10,873 INFO [M:0;c49966d8a95f:33041 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T03:46:10,873 DEBUG [M:0;c49966d8a95f:33041 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733888770784Disabling compacts and flushes for region at 1733888770784Disabling writes for close at 1733888770785 (+1 ms)Obtaining lock to block concurrent updates at 1733888770785Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733888770785Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1733888770785Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733888770786 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733888770786Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733888770808 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733888770808Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733888770812 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733888770832 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733888770832Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733888770836 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733888770858 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733888770858Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@488c0f79: reopening flushed file at 1733888770861 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ac1255f: reopening flushed file at 1733888770864 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@723b5221: reopening flushed file at 1733888770867 (+3 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 86ms, sequenceid=84, compaction requested=false at 1733888770871 (+4 ms)Writing region close event to WAL at 1733888770873 (+2 ms)Closed at 1733888770873 2024-12-11T03:46:10,875 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T03:46:10,875 INFO [M:0;c49966d8a95f:33041 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-11T03:46:10,875 INFO [M:0;c49966d8a95f:33041 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33041 2024-12-11T03:46:10,875 INFO [M:0;c49966d8a95f:33041 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T03:46:10,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T03:46:10,978 INFO [M:0;c49966d8a95f:33041 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-11T03:46:10,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33041-0x1019fa0d63e0000, quorum=127.0.0.1:57764, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T03:46:10,982 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-11T03:46:11,002 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-11T03:46:11,002 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:11,011 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733888770470 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733888768943 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/c49966d8a95f:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733888770640 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/c49966d8a95f:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/c49966d8a95f:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/c49966d8a95f:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: hfile-prefetch-1733888770637 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=184 (was 165) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4217 (was 4544) 2024-12-11T03:46:11,017 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=184, ProcessCount=11, AvailableMemoryMB=4218 2024-12-11T03:46:11,018 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/hadoop-log-dir so I do NOT create it in target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284 2024-12-11T03:46:11,018 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/hadoop-tmp-dir so I do NOT create it in target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284 2024-12-11T03:46:11,018 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:11,021 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284/cluster_211935a9-d958-a86a-f615-b92eb4d98b72, deleteOnExit=true 2024-12-11T03:46:11,024 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284/cluster_211935a9-d958-a86a-f615-b92eb4d98b72/zookeeper_0, clientPort=55193, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284/cluster_211935a9-d958-a86a-f615-b92eb4d98b72/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284/cluster_211935a9-d958-a86a-f615-b92eb4d98b72/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-11T03:46:11,025 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55193 2024-12-11T03:46:11,036 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e with version=8 2024-12-11T03:46:11,037 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/hbase-staging 2024-12-11T03:46:11,039 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c49966d8a95f:0 server-side Connection retries=45 2024-12-11T03:46:11,039 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:11,039 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:11,039 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T03:46:11,039 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:11,040 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T03:46:11,040 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-11T03:46:11,040 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T03:46:11,041 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34767 2024-12-11T03:46:11,042 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34767 connecting to ZooKeeper ensemble=127.0.0.1:55193 2024-12-11T03:46:11,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:347670x0, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T03:46:11,052 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34767-0x1019fa0ee5f0000 connected 2024-12-11T03:46:11,069 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:11,069 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:11,070 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T03:46:11,071 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e, hbase.cluster.distributed=false 2024-12-11T03:46:11,072 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T03:46:11,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34767 2024-12-11T03:46:11,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34767 2024-12-11T03:46:11,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34767 2024-12-11T03:46:11,074 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34767 2024-12-11T03:46:11,074 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34767 2024-12-11T03:46:11,090 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c49966d8a95f:0 server-side Connection retries=45 2024-12-11T03:46:11,090 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:11,090 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:11,090 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T03:46:11,090 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:11,090 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T03:46:11,090 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-11T03:46:11,091 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T03:46:11,091 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37977 2024-12-11T03:46:11,093 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37977 connecting to ZooKeeper ensemble=127.0.0.1:55193 2024-12-11T03:46:11,093 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:11,093 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:11,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:379770x0, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T03:46:11,097 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37977-0x1019fa0ee5f0001 connected 2024-12-11T03:46:11,097 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T03:46:11,097 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-11T03:46:11,098 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284/bucket.cache 2024-12-11T03:46:11,098 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-11T03:46:11,098 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-11T03:46:11,098 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-11T03:46:11,099 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-11T03:46:11,099 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-11T03:46:11,099 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-11T03:46:11,099 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-11T03:46:11,099 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-11T03:46:11,099 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284/bucket.persistence is complete. 2024-12-11T03:46:11,100 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-11T03:46:11,101 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T03:46:11,102 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37977 2024-12-11T03:46:11,103 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37977 2024-12-11T03:46:11,103 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37977 2024-12-11T03:46:11,103 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37977 2024-12-11T03:46:11,104 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37977 2024-12-11T03:46:11,120 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c49966d8a95f:0 server-side Connection retries=45 2024-12-11T03:46:11,120 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:11,120 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:11,120 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T03:46:11,120 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:11,120 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T03:46:11,120 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-11T03:46:11,120 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T03:46:11,121 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35587 2024-12-11T03:46:11,122 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35587 connecting to ZooKeeper ensemble=127.0.0.1:55193 2024-12-11T03:46:11,123 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:11,123 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:11,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:355870x0, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T03:46:11,129 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35587-0x1019fa0ee5f0002 connected 2024-12-11T03:46:11,129 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35587-0x1019fa0ee5f0002, quorum=127.0.0.1:55193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T03:46:11,130 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-11T03:46:11,130 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284/bucket.cache 2024-12-11T03:46:11,131 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-11T03:46:11,131 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-11T03:46:11,131 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-11T03:46:11,131 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-11T03:46:11,131 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-11T03:46:11,131 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-11T03:46:11,131 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-11T03:46:11,131 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284/bucket.persistence is complete. 2024-12-11T03:46:11,132 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-11T03:46:11,132 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35587-0x1019fa0ee5f0002, quorum=127.0.0.1:55193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-11T03:46:11,134 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35587-0x1019fa0ee5f0002, quorum=127.0.0.1:55193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T03:46:11,134 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35587 2024-12-11T03:46:11,134 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35587 2024-12-11T03:46:11,135 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35587 2024-12-11T03:46:11,135 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35587 2024-12-11T03:46:11,136 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35587 2024-12-11T03:46:11,149 DEBUG [M:0;c49966d8a95f:34767 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c49966d8a95f:34767 2024-12-11T03:46:11,149 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c49966d8a95f,34767,1733888771039 2024-12-11T03:46:11,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T03:46:11,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35587-0x1019fa0ee5f0002, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T03:46:11,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T03:46:11,154 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c49966d8a95f,34767,1733888771039 2024-12-11T03:46:11,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-11T03:46:11,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:11,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:11,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35587-0x1019fa0ee5f0002, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-11T03:46:11,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35587-0x1019fa0ee5f0002, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:11,158 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-11T03:46:11,158 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c49966d8a95f,34767,1733888771039 from backup master directory 2024-12-11T03:46:11,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c49966d8a95f,34767,1733888771039 2024-12-11T03:46:11,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35587-0x1019fa0ee5f0002, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T03:46:11,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T03:46:11,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T03:46:11,160 WARN [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T03:46:11,160 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c49966d8a95f,34767,1733888771039 2024-12-11T03:46:11,161 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/hbase.id] with ID: 28594740-525e-480c-9168-740b0ce236d8 2024-12-11T03:46:11,161 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/.tmp/hbase.id 2024-12-11T03:46:11,165 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/hbase.id] 2024-12-11T03:46:11,199 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:11,200 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-11T03:46:11,201 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-12-11T03:46:11,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:11,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:11,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35587-0x1019fa0ee5f0002, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:11,208 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T03:46:11,211 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-11T03:46:11,211 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T03:46:11,232 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store 2024-12-11T03:46:11,238 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T03:46:11,238 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-11T03:46:11,238 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T03:46:11,238 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T03:46:11,238 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-11T03:46:11,238 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T03:46:11,238 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T03:46:11,238 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733888771238Disabling compacts and flushes for region at 1733888771238Disabling writes for close at 1733888771238Writing region close event to WAL at 1733888771238Closed at 1733888771238 2024-12-11T03:46:11,239 WARN [master/c49966d8a95f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/.initializing 2024-12-11T03:46:11,239 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/WALs/c49966d8a95f,34767,1733888771039 2024-12-11T03:46:11,244 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T03:46:11,248 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c49966d8a95f%2C34767%2C1733888771039, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/WALs/c49966d8a95f,34767,1733888771039, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/oldWALs, maxLogs=10 2024-12-11T03:46:11,267 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/WALs/c49966d8a95f,34767,1733888771039/c49966d8a95f%2C34767%2C1733888771039.1733888771248 2024-12-11T03:46:11,268 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-11T03:46:11,268 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-11T03:46:11,268 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T03:46:11,268 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:11,268 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:11,269 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:11,274 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-11T03:46:11,274 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:11,274 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T03:46:11,275 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:11,278 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-11T03:46:11,278 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:11,279 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T03:46:11,279 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:11,283 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-11T03:46:11,283 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:11,283 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T03:46:11,283 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:11,287 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-11T03:46:11,287 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:11,287 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T03:46:11,287 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:11,288 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:11,288 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:11,288 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:11,288 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:11,288 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T03:46:11,289 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T03:46:11,295 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T03:46:11,296 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70367878, jitterRate=0.048563092947006226}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T03:46:11,297 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733888771268Initializing all the Stores at 1733888771269 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733888771269Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733888771269Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733888771269Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733888771269Cleaning up temporary data from old regions at 1733888771288 (+19 ms)Region opened successfully at 1733888771296 (+8 ms) 2024-12-11T03:46:11,297 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-11T03:46:11,301 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@461989c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c49966d8a95f/172.17.0.2:0 2024-12-11T03:46:11,303 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-11T03:46:11,303 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-11T03:46:11,303 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-11T03:46:11,303 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T03:46:11,303 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T03:46:11,304 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-11T03:46:11,304 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-11T03:46:11,305 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-11T03:46:11,306 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-11T03:46:11,309 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-11T03:46:11,309 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-11T03:46:11,310 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-11T03:46:11,311 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-11T03:46:11,312 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-11T03:46:11,312 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-11T03:46:11,314 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-11T03:46:11,314 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-11T03:46:11,316 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-11T03:46:11,318 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-11T03:46:11,325 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-11T03:46:11,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T03:46:11,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T03:46:11,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35587-0x1019fa0ee5f0002, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T03:46:11,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:11,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:11,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35587-0x1019fa0ee5f0002, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:11,328 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c49966d8a95f,34767,1733888771039, sessionid=0x1019fa0ee5f0000, setting cluster-up flag (Was=false) 2024-12-11T03:46:11,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:11,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35587-0x1019fa0ee5f0002, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:11,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:11,338 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-11T03:46:11,339 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c49966d8a95f,34767,1733888771039 2024-12-11T03:46:11,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:11,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35587-0x1019fa0ee5f0002, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:11,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:11,348 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-11T03:46:11,349 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c49966d8a95f,34767,1733888771039 2024-12-11T03:46:11,350 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-11T03:46:11,351 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T03:46:11,352 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-11T03:46:11,352 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-11T03:46:11,352 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c49966d8a95f,34767,1733888771039 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-11T03:46:11,354 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c49966d8a95f:0, corePoolSize=5, maxPoolSize=5 2024-12-11T03:46:11,354 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c49966d8a95f:0, corePoolSize=5, maxPoolSize=5 2024-12-11T03:46:11,354 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c49966d8a95f:0, corePoolSize=5, maxPoolSize=5 2024-12-11T03:46:11,354 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c49966d8a95f:0, corePoolSize=5, maxPoolSize=5 2024-12-11T03:46:11,354 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c49966d8a95f:0, corePoolSize=10, maxPoolSize=10 2024-12-11T03:46:11,354 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,354 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c49966d8a95f:0, corePoolSize=2, maxPoolSize=2 2024-12-11T03:46:11,354 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,356 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T03:46:11,356 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-11T03:46:11,357 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733888801356 2024-12-11T03:46:11,357 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-11T03:46:11,357 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-11T03:46:11,357 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-11T03:46:11,357 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-11T03:46:11,357 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:11,357 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-11T03:46:11,357 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-11T03:46:11,357 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,357 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-11T03:46:11,358 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-11T03:46:11,358 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-11T03:46:11,358 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-11T03:46:11,358 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-11T03:46:11,358 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-11T03:46:11,359 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c49966d8a95f:0:becomeActiveMaster-HFileCleaner.large.0-1733888771358,5,FailOnTimeoutGroup] 2024-12-11T03:46:11,360 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c49966d8a95f:0:becomeActiveMaster-HFileCleaner.small.0-1733888771359,5,FailOnTimeoutGroup] 2024-12-11T03:46:11,360 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,360 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-11T03:46:11,361 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,361 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,375 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-11T03:46:11,375 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e 2024-12-11T03:46:11,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T03:46:11,384 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-11T03:46:11,388 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-11T03:46:11,388 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:11,388 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T03:46:11,388 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-11T03:46:11,393 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-11T03:46:11,393 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:11,393 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T03:46:11,394 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-11T03:46:11,399 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-11T03:46:11,399 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:11,399 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T03:46:11,399 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-11T03:46:11,402 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-11T03:46:11,402 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:11,403 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T03:46:11,403 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-11T03:46:11,404 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740 2024-12-11T03:46:11,404 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740 2024-12-11T03:46:11,404 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-11T03:46:11,404 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-11T03:46:11,405 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T03:46:11,405 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-11T03:46:11,413 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T03:46:11,413 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58847240, jitterRate=-0.12310779094696045}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T03:46:11,414 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733888771383Initializing all the Stores at 1733888771383Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733888771383Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733888771383Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733888771384 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733888771384Cleaning up temporary data from old regions at 1733888771404 (+20 ms)Region opened successfully at 1733888771414 (+10 ms) 2024-12-11T03:46:11,414 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-11T03:46:11,414 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-11T03:46:11,414 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-11T03:46:11,414 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-11T03:46:11,414 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-11T03:46:11,415 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-11T03:46:11,415 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733888771414Disabling compacts and flushes for region at 1733888771414Disabling writes for close at 1733888771414Writing region close event to WAL at 1733888771415 (+1 ms)Closed at 1733888771415 2024-12-11T03:46:11,416 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T03:46:11,416 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-11T03:46:11,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-11T03:46:11,417 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T03:46:11,418 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-11T03:46:11,439 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.HRegionServer(746): ClusterId : 28594740-525e-480c-9168-740b0ce236d8 2024-12-11T03:46:11,439 DEBUG [RS:0;c49966d8a95f:37977 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-11T03:46:11,439 INFO [RS:1;c49966d8a95f:35587 {}] regionserver.HRegionServer(746): ClusterId : 28594740-525e-480c-9168-740b0ce236d8 2024-12-11T03:46:11,439 DEBUG [RS:1;c49966d8a95f:35587 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-11T03:46:11,444 DEBUG [RS:0;c49966d8a95f:37977 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-11T03:46:11,444 DEBUG [RS:1;c49966d8a95f:35587 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-11T03:46:11,444 DEBUG [RS:1;c49966d8a95f:35587 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-11T03:46:11,444 DEBUG [RS:0;c49966d8a95f:37977 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-11T03:46:11,446 DEBUG [RS:0;c49966d8a95f:37977 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-11T03:46:11,446 DEBUG [RS:1;c49966d8a95f:35587 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-11T03:46:11,447 DEBUG [RS:0;c49966d8a95f:37977 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d36a5f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c49966d8a95f/172.17.0.2:0 2024-12-11T03:46:11,447 DEBUG [RS:1;c49966d8a95f:35587 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65fc57dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c49966d8a95f/172.17.0.2:0 2024-12-11T03:46:11,461 DEBUG [RS:0;c49966d8a95f:37977 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c49966d8a95f:37977 2024-12-11T03:46:11,461 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-11T03:46:11,461 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-11T03:46:11,461 DEBUG [RS:0;c49966d8a95f:37977 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-11T03:46:11,462 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.HRegionServer(2659): reportForDuty to master=c49966d8a95f,34767,1733888771039 with port=37977, startcode=1733888771090 2024-12-11T03:46:11,462 DEBUG [RS:0;c49966d8a95f:37977 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-11T03:46:11,465 DEBUG [RS:1;c49966d8a95f:35587 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;c49966d8a95f:35587 2024-12-11T03:46:11,465 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56623, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-11T03:46:11,465 INFO [RS:1;c49966d8a95f:35587 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-11T03:46:11,465 INFO [RS:1;c49966d8a95f:35587 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-11T03:46:11,465 DEBUG [RS:1;c49966d8a95f:35587 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-11T03:46:11,466 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34767 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c49966d8a95f,37977,1733888771090 2024-12-11T03:46:11,466 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34767 {}] master.ServerManager(517): Registering regionserver=c49966d8a95f,37977,1733888771090 2024-12-11T03:46:11,466 INFO [RS:1;c49966d8a95f:35587 {}] regionserver.HRegionServer(2659): reportForDuty to master=c49966d8a95f,34767,1733888771039 with port=35587, startcode=1733888771119 2024-12-11T03:46:11,466 DEBUG [RS:1;c49966d8a95f:35587 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-11T03:46:11,467 DEBUG [RS:0;c49966d8a95f:37977 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e 2024-12-11T03:46:11,467 DEBUG [RS:0;c49966d8a95f:37977 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-11T03:46:11,468 DEBUG [RS:0;c49966d8a95f:37977 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-11T03:46:11,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T03:46:11,470 DEBUG [RS:0;c49966d8a95f:37977 {}] zookeeper.ZKUtil(111): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c49966d8a95f,37977,1733888771090 2024-12-11T03:46:11,470 WARN [RS:0;c49966d8a95f:37977 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T03:46:11,470 INFO [RS:0;c49966d8a95f:37977 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T03:46:11,470 DEBUG [RS:0;c49966d8a95f:37977 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,37977,1733888771090 2024-12-11T03:46:11,478 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c49966d8a95f,37977,1733888771090] 2024-12-11T03:46:11,481 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43729, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-11T03:46:11,482 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34767 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c49966d8a95f,35587,1733888771119 2024-12-11T03:46:11,482 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34767 {}] master.ServerManager(517): Registering regionserver=c49966d8a95f,35587,1733888771119 2024-12-11T03:46:11,482 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-11T03:46:11,484 DEBUG [RS:1;c49966d8a95f:35587 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e 2024-12-11T03:46:11,484 DEBUG [RS:1;c49966d8a95f:35587 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-11T03:46:11,484 DEBUG [RS:1;c49966d8a95f:35587 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-11T03:46:11,486 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-11T03:46:11,486 INFO [RS:0;c49966d8a95f:37977 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-11T03:46:11,486 INFO [RS:0;c49966d8a95f:37977 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,487 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-11T03:46:11,487 INFO [RS:0;c49966d8a95f:37977 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-11T03:46:11,487 INFO [RS:0;c49966d8a95f:37977 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,488 DEBUG [RS:0;c49966d8a95f:37977 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,488 DEBUG [RS:0;c49966d8a95f:37977 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,488 DEBUG [RS:0;c49966d8a95f:37977 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,488 DEBUG [RS:0;c49966d8a95f:37977 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,488 DEBUG [RS:0;c49966d8a95f:37977 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,488 DEBUG [RS:0;c49966d8a95f:37977 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c49966d8a95f:0, corePoolSize=2, maxPoolSize=2 2024-12-11T03:46:11,488 DEBUG [RS:0;c49966d8a95f:37977 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,488 DEBUG [RS:0;c49966d8a95f:37977 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,488 DEBUG [RS:0;c49966d8a95f:37977 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,488 DEBUG [RS:0;c49966d8a95f:37977 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,488 DEBUG [RS:0;c49966d8a95f:37977 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,489 DEBUG [RS:0;c49966d8a95f:37977 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,489 DEBUG [RS:0;c49966d8a95f:37977 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c49966d8a95f:0, corePoolSize=3, maxPoolSize=3 2024-12-11T03:46:11,489 DEBUG [RS:0;c49966d8a95f:37977 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c49966d8a95f:0, corePoolSize=3, maxPoolSize=3 2024-12-11T03:46:11,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T03:46:11,491 DEBUG [RS:1;c49966d8a95f:35587 {}] zookeeper.ZKUtil(111): regionserver:35587-0x1019fa0ee5f0002, quorum=127.0.0.1:55193, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c49966d8a95f,35587,1733888771119 2024-12-11T03:46:11,491 WARN [RS:1;c49966d8a95f:35587 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T03:46:11,491 INFO [RS:1;c49966d8a95f:35587 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T03:46:11,491 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c49966d8a95f,35587,1733888771119] 2024-12-11T03:46:11,491 DEBUG [RS:1;c49966d8a95f:35587 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,35587,1733888771119 2024-12-11T03:46:11,500 INFO [RS:0;c49966d8a95f:37977 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,500 INFO [RS:0;c49966d8a95f:37977 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,500 INFO [RS:0;c49966d8a95f:37977 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,500 INFO [RS:0;c49966d8a95f:37977 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,500 INFO [RS:0;c49966d8a95f:37977 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,500 INFO [RS:0;c49966d8a95f:37977 {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,37977,1733888771090-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T03:46:11,502 INFO [RS:1;c49966d8a95f:35587 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-11T03:46:11,505 INFO [RS:1;c49966d8a95f:35587 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-11T03:46:11,508 INFO [RS:1;c49966d8a95f:35587 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-11T03:46:11,509 INFO [RS:1;c49966d8a95f:35587 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,510 INFO [RS:1;c49966d8a95f:35587 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-11T03:46:11,510 INFO [RS:1;c49966d8a95f:35587 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-11T03:46:11,510 INFO [RS:1;c49966d8a95f:35587 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,510 DEBUG [RS:1;c49966d8a95f:35587 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,511 DEBUG [RS:1;c49966d8a95f:35587 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,511 DEBUG [RS:1;c49966d8a95f:35587 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,511 DEBUG [RS:1;c49966d8a95f:35587 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,511 DEBUG [RS:1;c49966d8a95f:35587 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,511 DEBUG [RS:1;c49966d8a95f:35587 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c49966d8a95f:0, corePoolSize=2, maxPoolSize=2 2024-12-11T03:46:11,511 DEBUG [RS:1;c49966d8a95f:35587 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,511 DEBUG [RS:1;c49966d8a95f:35587 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,511 DEBUG [RS:1;c49966d8a95f:35587 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,512 DEBUG [RS:1;c49966d8a95f:35587 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,512 DEBUG [RS:1;c49966d8a95f:35587 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,512 DEBUG [RS:1;c49966d8a95f:35587 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:11,512 DEBUG [RS:1;c49966d8a95f:35587 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c49966d8a95f:0, corePoolSize=3, maxPoolSize=3 2024-12-11T03:46:11,512 DEBUG [RS:1;c49966d8a95f:35587 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c49966d8a95f:0, corePoolSize=3, maxPoolSize=3 2024-12-11T03:46:11,513 INFO [RS:1;c49966d8a95f:35587 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,513 INFO [RS:1;c49966d8a95f:35587 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,513 INFO [RS:1;c49966d8a95f:35587 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,513 INFO [RS:1;c49966d8a95f:35587 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,513 INFO [RS:1;c49966d8a95f:35587 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,513 INFO [RS:1;c49966d8a95f:35587 {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,35587,1733888771119-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T03:46:11,517 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.HeapMemoryManager(213): Starting, tuneOn=false 2024-12-11T03:46:11,518 INFO [RS:0;c49966d8a95f:37977 {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,37977,1733888771090-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,518 INFO [RS:0;c49966d8a95f:37977 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,518 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.Replication(171): c49966d8a95f,37977,1733888771090 started 2024-12-11T03:46:11,528 INFO [RS:1;c49966d8a95f:35587 {}] regionserver.HeapMemoryManager(213): Starting, tuneOn=false 2024-12-11T03:46:11,529 INFO [RS:1;c49966d8a95f:35587 {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,35587,1733888771119-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,529 INFO [RS:1;c49966d8a95f:35587 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,529 INFO [RS:1;c49966d8a95f:35587 {}] regionserver.Replication(171): c49966d8a95f,35587,1733888771119 started 2024-12-11T03:46:11,533 INFO [RS:0;c49966d8a95f:37977 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,533 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.HRegionServer(1482): Serving as c49966d8a95f,37977,1733888771090, RpcServer on c49966d8a95f/172.17.0.2:37977, sessionid=0x1019fa0ee5f0001 2024-12-11T03:46:11,533 DEBUG [RS:0;c49966d8a95f:37977 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-11T03:46:11,533 DEBUG [RS:0;c49966d8a95f:37977 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c49966d8a95f,37977,1733888771090 2024-12-11T03:46:11,533 DEBUG [RS:0;c49966d8a95f:37977 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c49966d8a95f,37977,1733888771090' 2024-12-11T03:46:11,534 DEBUG [RS:0;c49966d8a95f:37977 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-11T03:46:11,534 DEBUG [RS:0;c49966d8a95f:37977 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-11T03:46:11,535 DEBUG [RS:0;c49966d8a95f:37977 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-11T03:46:11,535 DEBUG [RS:0;c49966d8a95f:37977 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-11T03:46:11,535 DEBUG [RS:0;c49966d8a95f:37977 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c49966d8a95f,37977,1733888771090 2024-12-11T03:46:11,535 DEBUG [RS:0;c49966d8a95f:37977 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c49966d8a95f,37977,1733888771090' 2024-12-11T03:46:11,535 DEBUG [RS:0;c49966d8a95f:37977 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-11T03:46:11,536 DEBUG [RS:0;c49966d8a95f:37977 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-11T03:46:11,536 DEBUG [RS:0;c49966d8a95f:37977 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-11T03:46:11,536 INFO [RS:0;c49966d8a95f:37977 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-11T03:46:11,536 INFO [RS:0;c49966d8a95f:37977 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-11T03:46:11,544 INFO [RS:1;c49966d8a95f:35587 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:11,544 INFO [RS:1;c49966d8a95f:35587 {}] regionserver.HRegionServer(1482): Serving as c49966d8a95f,35587,1733888771119, RpcServer on c49966d8a95f/172.17.0.2:35587, sessionid=0x1019fa0ee5f0002 2024-12-11T03:46:11,545 DEBUG [RS:1;c49966d8a95f:35587 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-11T03:46:11,545 DEBUG [RS:1;c49966d8a95f:35587 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c49966d8a95f,35587,1733888771119 2024-12-11T03:46:11,545 DEBUG [RS:1;c49966d8a95f:35587 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c49966d8a95f,35587,1733888771119' 2024-12-11T03:46:11,545 DEBUG [RS:1;c49966d8a95f:35587 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-11T03:46:11,545 DEBUG [RS:1;c49966d8a95f:35587 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-11T03:46:11,546 DEBUG [RS:1;c49966d8a95f:35587 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-11T03:46:11,546 DEBUG [RS:1;c49966d8a95f:35587 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-11T03:46:11,546 DEBUG [RS:1;c49966d8a95f:35587 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c49966d8a95f,35587,1733888771119 2024-12-11T03:46:11,546 DEBUG [RS:1;c49966d8a95f:35587 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c49966d8a95f,35587,1733888771119' 2024-12-11T03:46:11,546 DEBUG [RS:1;c49966d8a95f:35587 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-11T03:46:11,546 DEBUG [RS:1;c49966d8a95f:35587 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-11T03:46:11,547 DEBUG [RS:1;c49966d8a95f:35587 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-11T03:46:11,547 INFO [RS:1;c49966d8a95f:35587 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-11T03:46:11,547 INFO [RS:1;c49966d8a95f:35587 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-11T03:46:11,568 WARN [c49966d8a95f:34767 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T03:46:11,637 INFO [RS:0;c49966d8a95f:37977 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T03:46:11,638 INFO [RS:0;c49966d8a95f:37977 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c49966d8a95f%2C37977%2C1733888771090, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,37977,1733888771090, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/oldWALs, maxLogs=55 2024-12-11T03:46:11,648 INFO [RS:1;c49966d8a95f:35587 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T03:46:11,648 INFO [RS:1;c49966d8a95f:35587 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c49966d8a95f%2C35587%2C1733888771119, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,35587,1733888771119, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/oldWALs, maxLogs=55 2024-12-11T03:46:11,658 INFO [RS:0;c49966d8a95f:37977 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,37977,1733888771090/c49966d8a95f%2C37977%2C1733888771090.1733888771639 2024-12-11T03:46:11,658 DEBUG [RS:0;c49966d8a95f:37977 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-11T03:46:11,669 INFO [RS:1;c49966d8a95f:35587 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,35587,1733888771119/c49966d8a95f%2C35587%2C1733888771119.1733888771650 2024-12-11T03:46:11,669 DEBUG [RS:1;c49966d8a95f:35587 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-11T03:46:11,818 DEBUG [c49966d8a95f:34767 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-11T03:46:11,819 DEBUG [c49966d8a95f:34767 {}] balancer.BalancerClusterState(204): Hosts are {c49966d8a95f=0} racks are {/default-rack=0} 2024-12-11T03:46:11,820 DEBUG [c49966d8a95f:34767 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-11T03:46:11,820 DEBUG [c49966d8a95f:34767 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-11T03:46:11,820 DEBUG [c49966d8a95f:34767 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-11T03:46:11,820 DEBUG [c49966d8a95f:34767 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-11T03:46:11,820 INFO [c49966d8a95f:34767 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-11T03:46:11,820 INFO [c49966d8a95f:34767 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-11T03:46:11,820 DEBUG [c49966d8a95f:34767 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-11T03:46:11,820 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c49966d8a95f,37977,1733888771090 2024-12-11T03:46:11,821 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c49966d8a95f,37977,1733888771090, state=OPENING 2024-12-11T03:46:11,823 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-11T03:46:11,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:11,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:11,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35587-0x1019fa0ee5f0002, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:11,825 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T03:46:11,825 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T03:46:11,825 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c49966d8a95f,37977,1733888771090}] 2024-12-11T03:46:11,825 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T03:46:11,825 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T03:46:11,978 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-11T03:46:11,980 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45657, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-11T03:46:11,982 INFO [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-11T03:46:11,982 INFO [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T03:46:11,983 INFO [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-11T03:46:11,984 INFO [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c49966d8a95f%2C37977%2C1733888771090.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,37977,1733888771090, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/oldWALs, maxLogs=55 2024-12-11T03:46:12,003 INFO [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,37977,1733888771090/c49966d8a95f%2C37977%2C1733888771090.meta.1733888771984.meta 2024-12-11T03:46:12,004 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-11T03:46:12,004 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-11T03:46:12,004 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-11T03:46:12,004 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-11T03:46:12,004 INFO [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-11T03:46:12,005 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-11T03:46:12,005 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T03:46:12,005 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-11T03:46:12,005 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-11T03:46:12,006 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-11T03:46:12,006 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-11T03:46:12,006 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:12,007 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T03:46:12,007 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-11T03:46:12,007 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-11T03:46:12,007 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:12,008 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T03:46:12,008 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-11T03:46:12,008 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-11T03:46:12,008 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:12,008 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T03:46:12,009 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-11T03:46:12,009 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-11T03:46:12,009 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:12,009 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T03:46:12,009 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-11T03:46:12,010 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740 2024-12-11T03:46:12,010 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740 2024-12-11T03:46:12,011 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-11T03:46:12,011 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-11T03:46:12,011 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T03:46:12,011 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-11T03:46:12,012 INFO [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73598347, jitterRate=0.09670083224773407}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T03:46:12,012 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-11T03:46:12,012 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733888772005Writing region info on filesystem at 1733888772005Initializing all the Stores at 1733888772005Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733888772005Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733888772005Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733888772005Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733888772005Cleaning up temporary data from old regions at 1733888772011 (+6 ms)Running coprocessor post-open hooks at 1733888772012 (+1 ms)Region opened successfully at 1733888772012 2024-12-11T03:46:12,013 INFO [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733888771978 2024-12-11T03:46:12,016 DEBUG [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-11T03:46:12,016 INFO [RS_OPEN_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-11T03:46:12,016 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c49966d8a95f,37977,1733888771090 2024-12-11T03:46:12,017 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c49966d8a95f,37977,1733888771090, state=OPEN 2024-12-11T03:46:12,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T03:46:12,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T03:46:12,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35587-0x1019fa0ee5f0002, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T03:46:12,019 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T03:46:12,019 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c49966d8a95f,37977,1733888771090 2024-12-11T03:46:12,019 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T03:46:12,019 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T03:46:12,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-11T03:46:12,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c49966d8a95f,37977,1733888771090 in 194 msec 2024-12-11T03:46:12,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-11T03:46:12,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 605 msec 2024-12-11T03:46:12,024 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T03:46:12,024 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-11T03:46:12,027 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T03:46:12,028 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c49966d8a95f,37977,1733888771090, seqNum=-1] 2024-12-11T03:46:12,028 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T03:46:12,030 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51005, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T03:46:12,035 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 684 msec 2024-12-11T03:46:12,036 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733888772036, completionTime=-1 2024-12-11T03:46:12,036 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-11T03:46:12,036 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T03:46:12,039 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T03:46:12,039 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733888832039 2024-12-11T03:46:12,039 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733888892039 2024-12-11T03:46:12,039 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-11T03:46:12,039 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,34767,1733888771039-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:12,040 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,34767,1733888771039-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:12,040 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,34767,1733888771039-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:12,040 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c49966d8a95f:34767, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:12,040 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:12,040 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:12,043 DEBUG [master/c49966d8a95f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-11T03:46:12,046 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.886sec 2024-12-11T03:46:12,046 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-11T03:46:12,046 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-11T03:46:12,046 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-11T03:46:12,046 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-11T03:46:12,047 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-11T03:46:12,047 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,34767,1733888771039-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T03:46:12,047 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,34767,1733888771039-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-11T03:46:12,048 DEBUG [master/c49966d8a95f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-11T03:46:12,048 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-11T03:46:12,048 INFO [master/c49966d8a95f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,34767,1733888771039-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:12,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31597363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T03:46:12,139 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c49966d8a95f,34767,-1 for getting cluster id 2024-12-11T03:46:12,139 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T03:46:12,141 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '28594740-525e-480c-9168-740b0ce236d8' 2024-12-11T03:46:12,142 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T03:46:12,142 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "28594740-525e-480c-9168-740b0ce236d8" 2024-12-11T03:46:12,142 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@792e4a8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T03:46:12,142 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c49966d8a95f,34767,-1] 2024-12-11T03:46:12,142 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T03:46:12,143 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T03:46:12,145 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T03:46:12,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31deb7f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T03:46:12,147 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T03:46:12,149 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c49966d8a95f,37977,1733888771090, seqNum=-1] 2024-12-11T03:46:12,150 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T03:46:12,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T03:46:12,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c49966d8a95f,34767,1733888771039 2024-12-11T03:46:12,156 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T03:46:12,157 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is c49966d8a95f,34767,1733888771039 2024-12-11T03:46:12,157 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1ff3d236 2024-12-11T03:46:12,158 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T03:46:12,160 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T03:46:12,161 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34767 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733888772155', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T03:46:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34767 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733888772155 2024-12-11T03:46:12,163 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733888772155 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T03:46:12,163 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:12,163 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34767 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733888772155" procId is: 4 2024-12-11T03:46:12,164 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733888772155 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T03:46:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34767 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T03:46:12,176 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733888772155-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 34d44a2e3ad1c784cab546a4453e1557, NAME => 'testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733888772155', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e 2024-12-11T03:46:12,182 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733888772155-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T03:46:12,182 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733888772155-pool-0 {}] regionserver.HRegion(1722): Closing 34d44a2e3ad1c784cab546a4453e1557, disabling compactions & flushes 2024-12-11T03:46:12,183 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733888772155-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. 2024-12-11T03:46:12,183 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733888772155-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. 2024-12-11T03:46:12,183 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733888772155-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. after waiting 0 ms 2024-12-11T03:46:12,183 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733888772155-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. 2024-12-11T03:46:12,183 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733888772155-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. 2024-12-11T03:46:12,183 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733888772155-pool-0 {}] regionserver.HRegion(1676): Region close journal for 34d44a2e3ad1c784cab546a4453e1557: Waiting for close lock at 1733888772182Disabling compacts and flushes for region at 1733888772182Disabling writes for close at 1733888772183 (+1 ms)Writing region close event to WAL at 1733888772183Closed at 1733888772183 2024-12-11T03:46:12,184 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733888772155 execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T03:46:12,184 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733888772184"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733888772184"}]},"ts":"1733888772184"} 2024-12-11T03:46:12,186 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T03:46:12,187 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733888772155 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T03:46:12,187 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733888772155","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733888772187"}]},"ts":"1733888772187"} 2024-12-11T03:46:12,189 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733888772155, state=ENABLING in hbase:meta 2024-12-11T03:46:12,190 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {c49966d8a95f=0} racks are {/default-rack=0} 2024-12-11T03:46:12,190 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-11T03:46:12,190 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-11T03:46:12,190 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-11T03:46:12,190 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-11T03:46:12,190 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-11T03:46:12,190 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-11T03:46:12,190 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-11T03:46:12,191 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733888772155, region=34d44a2e3ad1c784cab546a4453e1557, ASSIGN}] 2024-12-11T03:46:12,191 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733888772155, region=34d44a2e3ad1c784cab546a4453e1557, ASSIGN 2024-12-11T03:46:12,192 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733888772155, region=34d44a2e3ad1c784cab546a4453e1557, ASSIGN; state=OFFLINE, location=c49966d8a95f,35587,1733888771119; forceNewPlan=false, retain=false 2024-12-11T03:46:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34767 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T03:46:12,343 INFO [c49966d8a95f:34767 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-11T03:46:12,343 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=34d44a2e3ad1c784cab546a4453e1557, regionState=OPENING, regionLocation=c49966d8a95f,35587,1733888771119 2024-12-11T03:46:12,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733888772155, region=34d44a2e3ad1c784cab546a4453e1557, ASSIGN because future has completed 2024-12-11T03:46:12,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34d44a2e3ad1c784cab546a4453e1557, server=c49966d8a95f,35587,1733888771119}] 2024-12-11T03:46:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34767 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T03:46:12,499 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-11T03:46:12,501 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60453, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-11T03:46:12,503 INFO [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. 2024-12-11T03:46:12,503 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 34d44a2e3ad1c784cab546a4453e1557, NAME => 'testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557.', STARTKEY => '', ENDKEY => ''} 2024-12-11T03:46:12,503 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733888772155 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:12,504 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T03:46:12,504 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:12,504 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:12,504 INFO [StoreOpener-34d44a2e3ad1c784cab546a4453e1557-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:12,509 INFO [StoreOpener-34d44a2e3ad1c784cab546a4453e1557-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34d44a2e3ad1c784cab546a4453e1557 columnFamilyName family 2024-12-11T03:46:12,509 DEBUG [StoreOpener-34d44a2e3ad1c784cab546a4453e1557-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:12,510 INFO [StoreOpener-34d44a2e3ad1c784cab546a4453e1557-1 {}] regionserver.HStore(327): Store=34d44a2e3ad1c784cab546a4453e1557/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T03:46:12,510 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:12,510 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:12,511 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:12,511 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:12,511 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:12,511 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:12,518 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T03:46:12,518 INFO [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 34d44a2e3ad1c784cab546a4453e1557; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62988462, jitterRate=-0.06139877438545227}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T03:46:12,518 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:12,519 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 34d44a2e3ad1c784cab546a4453e1557: Running coprocessor pre-open hook at 1733888772504Writing region info on filesystem at 1733888772504Initializing all the Stores at 1733888772504Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733888772504Cleaning up temporary data from old regions at 1733888772511 (+7 ms)Running coprocessor post-open hooks at 1733888772518 (+7 ms)Region opened successfully at 1733888772519 (+1 ms) 2024-12-11T03:46:12,520 INFO [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557., pid=6, masterSystemTime=1733888772499 2024-12-11T03:46:12,522 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. 2024-12-11T03:46:12,522 INFO [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. 2024-12-11T03:46:12,523 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=34d44a2e3ad1c784cab546a4453e1557, regionState=OPEN, openSeqNum=2, regionLocation=c49966d8a95f,35587,1733888771119 2024-12-11T03:46:12,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34d44a2e3ad1c784cab546a4453e1557, server=c49966d8a95f,35587,1733888771119 because future has completed 2024-12-11T03:46:12,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-11T03:46:12,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 34d44a2e3ad1c784cab546a4453e1557, server=c49966d8a95f,35587,1733888771119 in 179 msec 2024-12-11T03:46:12,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-11T03:46:12,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733888772155, region=34d44a2e3ad1c784cab546a4453e1557, ASSIGN in 338 msec 2024-12-11T03:46:12,531 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733888772155 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T03:46:12,531 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733888772155","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733888772531"}]},"ts":"1733888772531"} 2024-12-11T03:46:12,533 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733888772155, state=ENABLED in hbase:meta 2024-12-11T03:46:12,534 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733888772155 execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T03:46:12,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733888772155 in 373 msec 2024-12-11T03:46:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34767 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T03:46:12,798 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733888772155 completed 2024-12-11T03:46:12,798 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733888772155 get assigned. Timeout = 60000ms 2024-12-11T03:46:12,798 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T03:46:12,801 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733888772155 assigned to meta. Checking AM states. 2024-12-11T03:46:12,801 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T03:46:12,801 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733888772155 assigned. 2024-12-11T03:46:12,805 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733888772155', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557., hostname=c49966d8a95f,35587,1733888771119, seqNum=2] 2024-12-11T03:46:12,805 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T03:46:12,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T03:46:12,811 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 34d44a2e3ad1c784cab546a4453e1557 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-11T03:46:12,822 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 037b1277e2cf407bb0d03379c2aba103 2024-12-11T03:46:12,834 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557/.tmp/family/037b1277e2cf407bb0d03379c2aba103 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-11T03:46:12,835 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557/.tmp/family/037b1277e2cf407bb0d03379c2aba103 2024-12-11T03:46:12,838 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557/.tmp/family/037b1277e2cf407bb0d03379c2aba103 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557/family/037b1277e2cf407bb0d03379c2aba103 2024-12-11T03:46:12,839 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557/family/037b1277e2cf407bb0d03379c2aba103, delay=997 ms 2024-12-11T03:46:12,840 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557/family/037b1277e2cf407bb0d03379c2aba103, entries=2, sequenceid=6, filesize=4.9 K 2024-12-11T03:46:12,841 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 34d44a2e3ad1c784cab546a4453e1557 in 29ms, sequenceid=6, compaction requested=false 2024-12-11T03:46:12,841 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733888772155' 2024-12-11T03:46:12,841 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 34d44a2e3ad1c784cab546a4453e1557: 2024-12-11T03:46:13,293 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T03:46:13,837 DEBUG [hfile-prefetch-1733888770470 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 037b1277e2cf407bb0d03379c2aba103_0. 2024-12-11T03:46:13,837 DEBUG [hfile-prefetch-1733888770470 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 037b1277e2cf407bb0d03379c2aba103_117. 2024-12-11T03:46:13,838 DEBUG [hfile-prefetch-1733888770470 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557/family/037b1277e2cf407bb0d03379c2aba103, with total blocks 2, and data blocks 1 2024-12-11T03:46:13,838 DEBUG [hfile-prefetch-1733888770470 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-11T03:46:13,838 DEBUG [hfile-prefetch-1733888770470 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557/family/037b1277e2cf407bb0d03379c2aba103, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-11T03:46:13,838 DEBUG [hfile-prefetch-1733888770470 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557/family/037b1277e2cf407bb0d03379c2aba103. Will sleep 100ms and try the verification again. 2024-12-11T03:46:13,841 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;c49966d8a95f:35587,5,FailOnTimeoutGroup] 2024-12-11T03:46:13,842 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c49966d8a95f,35587,1733888771119' ***** 2024-12-11T03:46:13,842 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-12-11T03:46:13,842 INFO [RS:1;c49966d8a95f:35587 {}] bucket.BucketCache(1738): Disabling cache 2024-12-11T03:46:13,842 INFO [RS:1;c49966d8a95f:35587 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284/bucket.persistence 2024-12-11T03:46:13,842 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-11T03:46:13,842 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-11T03:46:13,842 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-11T03:46:13,842 INFO [RS:1;c49966d8a95f:35587 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-11T03:46:13,843 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-11T03:46:13,853 DEBUG [RS:1;c49966d8a95f:35587 {}] bucket.BucketCache(1487): Thread RS:1;c49966d8a95f:35587 started persisting bucket cache to file 2024-12-11T03:46:13,853 DEBUG [RS:1;c49966d8a95f:35587 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T03:46:13,853 DEBUG [RS:1;c49966d8a95f:35587 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-11T03:46:13,860 DEBUG [RS:1;c49966d8a95f:35587 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-11T03:46:13,860 DEBUG [RS:1;c49966d8a95f:35587 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733888773853 2024-12-11T03:46:13,860 DEBUG [RS:1;c49966d8a95f:35587 {}] bucket.BucketCache(1509): Thread RS:1;c49966d8a95f:35587 finished persisting bucket cache to file, renaming 2024-12-11T03:46:13,860 INFO [RS:1;c49966d8a95f:35587 {}] regionserver.HeapMemoryManager(224): Stopping 2024-12-11T03:46:13,861 INFO [RS:1;c49966d8a95f:35587 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-11T03:46:13,861 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-11T03:46:13,861 INFO [RS:1;c49966d8a95f:35587 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-11T03:46:13,861 INFO [RS:1;c49966d8a95f:35587 {}] regionserver.HRegionServer(3091): Received CLOSE for 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:13,861 INFO [RS:1;c49966d8a95f:35587 {}] regionserver.HRegionServer(959): stopping server c49966d8a95f,35587,1733888771119 2024-12-11T03:46:13,861 INFO [RS:1;c49966d8a95f:35587 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T03:46:13,861 INFO [RS:1;c49966d8a95f:35587 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;c49966d8a95f:35587. 2024-12-11T03:46:13,861 INFO [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-11T03:46:13,861 DEBUG [RS:1;c49966d8a95f:35587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T03:46:13,861 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 34d44a2e3ad1c784cab546a4453e1557, disabling compactions & flushes 2024-12-11T03:46:13,861 DEBUG [RS:1;c49966d8a95f:35587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T03:46:13,861 INFO [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. 2024-12-11T03:46:13,861 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. 2024-12-11T03:46:13,861 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. after waiting 0 ms 2024-12-11T03:46:13,861 INFO [RS:1;c49966d8a95f:35587 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-11T03:46:13,862 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. 2024-12-11T03:46:13,862 DEBUG [RS:1;c49966d8a95f:35587 {}] regionserver.HRegionServer(1325): Online Regions={34d44a2e3ad1c784cab546a4453e1557=testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557.} 2024-12-11T03:46:13,862 DEBUG [RS:1;c49966d8a95f:35587 {}] regionserver.HRegionServer(1351): Waiting on 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:13,862 DEBUG [StoreFileCloser-34d44a2e3ad1c784cab546a4453e1557-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557/family/037b1277e2cf407bb0d03379c2aba103 2024-12-11T03:46:13,862 WARN [hfile-prefetch-1733888770470 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557/family/037b1277e2cf407bb0d03379c2aba103, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-11T03:46:13,863 DEBUG [hfile-prefetch-1733888770470 {}] hfile.PrefetchExecutor(121): Prefetch completed for 037b1277e2cf407bb0d03379c2aba103 2024-12-11T03:46:13,867 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-11T03:46:13,867 INFO [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. 2024-12-11T03:46:13,867 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 34d44a2e3ad1c784cab546a4453e1557: Waiting for close lock at 1733888773861Running coprocessor pre-close hooks at 1733888773861Disabling compacts and flushes for region at 1733888773861Disabling writes for close at 1733888773862 (+1 ms)Writing region close event to WAL at 1733888773862Running coprocessor post-close hooks at 1733888773867 (+5 ms)Closed at 1733888773867 2024-12-11T03:46:13,867 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. 2024-12-11T03:46:13,901 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-11T03:46:13,901 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-11T03:46:13,901 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T03:46:13,901 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-11T03:46:13,904 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-11T03:46:13,904 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733888773901 2024-12-11T03:46:13,904 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-11T03:46:13,915 INFO [regionserver/c49966d8a95f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T03:46:14,062 INFO [RS:1;c49966d8a95f:35587 {}] regionserver.HRegionServer(976): stopping server c49966d8a95f,35587,1733888771119; all regions closed. 2024-12-11T03:46:14,064 DEBUG [RS:1;c49966d8a95f:35587 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/oldWALs 2024-12-11T03:46:14,064 INFO [RS:1;c49966d8a95f:35587 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c49966d8a95f%2C35587%2C1733888771119:(num 1733888771650) 2024-12-11T03:46:14,064 DEBUG [RS:1;c49966d8a95f:35587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T03:46:14,064 INFO [RS:1;c49966d8a95f:35587 {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T03:46:14,064 INFO [RS:1;c49966d8a95f:35587 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T03:46:14,064 INFO [RS:1;c49966d8a95f:35587 {}] hbase.ChoreService(370): Chore service for: regionserver/c49966d8a95f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-11T03:46:14,064 INFO [RS:1;c49966d8a95f:35587 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-11T03:46:14,065 INFO [regionserver/c49966d8a95f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T03:46:14,065 INFO [RS:1;c49966d8a95f:35587 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-11T03:46:14,065 INFO [RS:1;c49966d8a95f:35587 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-11T03:46:14,065 INFO [RS:1;c49966d8a95f:35587 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T03:46:14,065 INFO [RS:1;c49966d8a95f:35587 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35587 2024-12-11T03:46:14,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T03:46:14,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35587-0x1019fa0ee5f0002, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c49966d8a95f,35587,1733888771119 2024-12-11T03:46:14,069 INFO [RS:1;c49966d8a95f:35587 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T03:46:14,071 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c49966d8a95f,35587,1733888771119] 2024-12-11T03:46:14,075 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c49966d8a95f,35587,1733888771119 already deleted, retry=false 2024-12-11T03:46:14,075 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of c49966d8a95f,35587,1733888771119 on c49966d8a95f,34767,1733888771039 2024-12-11T03:46:14,079 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure c49966d8a95f,35587,1733888771119, splitWal=true, meta=false 2024-12-11T03:46:14,081 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for c49966d8a95f,35587,1733888771119 (carryingMeta=false) c49966d8a95f,35587,1733888771119/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@778117c8[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-11T03:46:14,082 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure c49966d8a95f,35587,1733888771119, splitWal=true, meta=false 2024-12-11T03:46:14,083 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): c49966d8a95f,35587,1733888771119 had 1 regions 2024-12-11T03:46:14,083 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure c49966d8a95f,35587,1733888771119, splitWal=true, meta=false, isMeta: false 2024-12-11T03:46:14,084 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,35587,1733888771119-splitting 2024-12-11T03:46:14,084 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,35587,1733888771119-splitting dir is empty, no logs to split. 2024-12-11T03:46:14,084 INFO [PEWorker-2 {}] master.SplitWALManager(105): c49966d8a95f,35587,1733888771119 WAL count=0, meta=false 2024-12-11T03:46:14,086 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,35587,1733888771119-splitting dir is empty, no logs to split. 2024-12-11T03:46:14,086 INFO [PEWorker-2 {}] master.SplitWALManager(105): c49966d8a95f,35587,1733888771119 WAL count=0, meta=false 2024-12-11T03:46:14,086 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if c49966d8a95f,35587,1733888771119 WAL splitting is done? wals=0, meta=false 2024-12-11T03:46:14,087 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for c49966d8a95f,35587,1733888771119 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,35587,1733888771119-splitting does not exist 2024-12-11T03:46:14,088 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733888772155, region=34d44a2e3ad1c784cab546a4453e1557, ASSIGN}] 2024-12-11T03:46:14,089 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733888772155, region=34d44a2e3ad1c784cab546a4453e1557, ASSIGN 2024-12-11T03:46:14,089 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733888772155, region=34d44a2e3ad1c784cab546a4453e1557, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-11T03:46:14,171 INFO [RS:1;c49966d8a95f:35587 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-11T03:46:14,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35587-0x1019fa0ee5f0002, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T03:46:14,171 INFO [RS:1;c49966d8a95f:35587 {}] regionserver.HRegionServer(1031): Exiting; stopping=c49966d8a95f,35587,1733888771119; zookeeper connection closed. 2024-12-11T03:46:14,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35587-0x1019fa0ee5f0002, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T03:46:14,172 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2e8db77f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2e8db77f 2024-12-11T03:46:14,240 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=34d44a2e3ad1c784cab546a4453e1557, regionState=OPENING, regionLocation=c49966d8a95f,37977,1733888771090 2024-12-11T03:46:14,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733888772155, region=34d44a2e3ad1c784cab546a4453e1557, ASSIGN because future has completed 2024-12-11T03:46:14,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34d44a2e3ad1c784cab546a4453e1557, server=c49966d8a95f,37977,1733888771090}] 2024-12-11T03:46:14,358 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c49966d8a95f:0 server-side Connection retries=45 2024-12-11T03:46:14,358 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:14,358 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:14,358 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T03:46:14,359 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T03:46:14,359 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T03:46:14,359 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-11T03:46:14,359 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T03:46:14,360 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43237 2024-12-11T03:46:14,361 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43237 connecting to ZooKeeper ensemble=127.0.0.1:55193 2024-12-11T03:46:14,361 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:14,361 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:14,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:432370x0, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T03:46:14,365 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43237-0x1019fa0ee5f0003 connected 2024-12-11T03:46:14,365 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:43237-0x1019fa0ee5f0003, quorum=127.0.0.1:55193, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-11T03:46:14,365 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-11T03:46:14,366 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-11T03:46:14,366 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284/bucket.cache 2024-12-11T03:46:14,366 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-11T03:46:14,367 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-11T03:46:14,367 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-11T03:46:14,367 INFO [Thread-200 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-11T03:46:14,368 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-11T03:46:14,368 INFO [Thread-200 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-11T03:46:14,368 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-11T03:46:14,368 INFO [Thread-200 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-12-11T03:46:14,369 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:43237-0x1019fa0ee5f0003, quorum=127.0.0.1:55193, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-11T03:46:14,370 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43237-0x1019fa0ee5f0003, quorum=127.0.0.1:55193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T03:46:14,371 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43237 2024-12-11T03:46:14,371 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43237 2024-12-11T03:46:14,372 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43237 2024-12-11T03:46:14,372 DEBUG [Thread-200 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-11T03:46:14,373 INFO [Thread-200 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-11T03:46:14,373 INFO [Thread-200 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-12-11T03:46:14,373 INFO [Thread-200 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284/bucket.persistence is complete. 2024-12-11T03:46:14,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43237 2024-12-11T03:46:14,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43237 2024-12-11T03:46:14,379 INFO [RS:2;c49966d8a95f:43237 {}] regionserver.HRegionServer(746): ClusterId : 28594740-525e-480c-9168-740b0ce236d8 2024-12-11T03:46:14,380 DEBUG [RS:2;c49966d8a95f:43237 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-11T03:46:14,382 DEBUG [RS:2;c49966d8a95f:43237 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-11T03:46:14,382 DEBUG [RS:2;c49966d8a95f:43237 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-11T03:46:14,384 DEBUG [RS:2;c49966d8a95f:43237 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-11T03:46:14,385 DEBUG [RS:2;c49966d8a95f:43237 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@171af37c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c49966d8a95f/172.17.0.2:0 2024-12-11T03:46:14,398 INFO [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. 2024-12-11T03:46:14,398 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 34d44a2e3ad1c784cab546a4453e1557, NAME => 'testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557.', STARTKEY => '', ENDKEY => ''} 2024-12-11T03:46:14,399 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733888772155 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:14,399 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T03:46:14,399 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:14,399 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:14,400 INFO [StoreOpener-34d44a2e3ad1c784cab546a4453e1557-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:14,400 INFO [StoreOpener-34d44a2e3ad1c784cab546a4453e1557-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34d44a2e3ad1c784cab546a4453e1557 columnFamilyName family 2024-12-11T03:46:14,401 DEBUG [StoreOpener-34d44a2e3ad1c784cab546a4453e1557-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T03:46:14,402 DEBUG [StoreFileOpener-34d44a2e3ad1c784cab546a4453e1557-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557/family/037b1277e2cf407bb0d03379c2aba103, delay=962 ms 2024-12-11T03:46:14,403 DEBUG [StoreOpener-34d44a2e3ad1c784cab546a4453e1557-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557/family/037b1277e2cf407bb0d03379c2aba103 2024-12-11T03:46:14,403 INFO [StoreOpener-34d44a2e3ad1c784cab546a4453e1557-1 {}] regionserver.HStore(327): Store=34d44a2e3ad1c784cab546a4453e1557/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T03:46:14,404 DEBUG [RS:2;c49966d8a95f:43237 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;c49966d8a95f:43237 2024-12-11T03:46:14,404 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:14,404 INFO [RS:2;c49966d8a95f:43237 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-11T03:46:14,404 INFO [RS:2;c49966d8a95f:43237 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-11T03:46:14,404 DEBUG [RS:2;c49966d8a95f:43237 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-11T03:46:14,404 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:14,405 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:14,405 INFO [RS:2;c49966d8a95f:43237 {}] regionserver.HRegionServer(2659): reportForDuty to master=c49966d8a95f,34767,1733888771039 with port=43237, startcode=1733888774358 2024-12-11T03:46:14,405 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:14,405 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:14,405 DEBUG [RS:2;c49966d8a95f:43237 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-11T03:46:14,406 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:14,407 INFO [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 34d44a2e3ad1c784cab546a4453e1557; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70174460, jitterRate=0.0456809401512146}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T03:46:14,407 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:14,408 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56037, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-11T03:46:14,408 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 34d44a2e3ad1c784cab546a4453e1557: Running coprocessor pre-open hook at 1733888774399Writing region info on filesystem at 1733888774399Initializing all the Stores at 1733888774399Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733888774399Cleaning up temporary data from old regions at 1733888774405 (+6 ms)Running coprocessor post-open hooks at 1733888774407 (+2 ms)Region opened successfully at 1733888774408 (+1 ms) 2024-12-11T03:46:14,408 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34767 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c49966d8a95f,43237,1733888774358 2024-12-11T03:46:14,408 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34767 {}] master.ServerManager(517): Registering regionserver=c49966d8a95f,43237,1733888774358 2024-12-11T03:46:14,409 INFO [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557., pid=9, masterSystemTime=1733888774396 2024-12-11T03:46:14,410 DEBUG [RS:2;c49966d8a95f:43237 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e 2024-12-11T03:46:14,410 DEBUG [RS:2;c49966d8a95f:43237 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-11T03:46:14,410 DEBUG [RS:2;c49966d8a95f:43237 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-11T03:46:14,411 DEBUG [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. 2024-12-11T03:46:14,411 INFO [RS_OPEN_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. 2024-12-11T03:46:14,412 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=34d44a2e3ad1c784cab546a4453e1557, regionState=OPEN, openSeqNum=10, regionLocation=c49966d8a95f,37977,1733888771090 2024-12-11T03:46:14,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T03:46:14,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34d44a2e3ad1c784cab546a4453e1557, server=c49966d8a95f,37977,1733888771090 because future has completed 2024-12-11T03:46:14,415 DEBUG [RS:2;c49966d8a95f:43237 {}] zookeeper.ZKUtil(111): regionserver:43237-0x1019fa0ee5f0003, quorum=127.0.0.1:55193, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c49966d8a95f,43237,1733888774358 2024-12-11T03:46:14,415 WARN [RS:2;c49966d8a95f:43237 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T03:46:14,415 INFO [RS:2;c49966d8a95f:43237 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T03:46:14,415 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c49966d8a95f,43237,1733888774358] 2024-12-11T03:46:14,415 DEBUG [RS:2;c49966d8a95f:43237 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,43237,1733888774358 2024-12-11T03:46:14,418 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-11T03:46:14,418 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 34d44a2e3ad1c784cab546a4453e1557, server=c49966d8a95f,37977,1733888771090 in 172 msec 2024-12-11T03:46:14,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-11T03:46:14,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733888772155, region=34d44a2e3ad1c784cab546a4453e1557, ASSIGN in 330 msec 2024-12-11T03:46:14,420 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server c49966d8a95f,35587,1733888771119 after splitting done 2024-12-11T03:46:14,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure c49966d8a95f,35587,1733888771119, splitWal=true, meta=false in 344 msec 2024-12-11T03:46:14,424 INFO [RS:2;c49966d8a95f:43237 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-11T03:46:14,426 INFO [RS:2;c49966d8a95f:43237 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-11T03:46:14,428 INFO [RS:2;c49966d8a95f:43237 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-11T03:46:14,428 INFO [RS:2;c49966d8a95f:43237 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:14,429 INFO [RS:2;c49966d8a95f:43237 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-11T03:46:14,430 INFO [RS:2;c49966d8a95f:43237 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-11T03:46:14,430 INFO [RS:2;c49966d8a95f:43237 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:14,430 DEBUG [RS:2;c49966d8a95f:43237 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:14,430 DEBUG [RS:2;c49966d8a95f:43237 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:14,430 DEBUG [RS:2;c49966d8a95f:43237 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:14,430 DEBUG [RS:2;c49966d8a95f:43237 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:14,430 DEBUG [RS:2;c49966d8a95f:43237 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:14,430 DEBUG [RS:2;c49966d8a95f:43237 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c49966d8a95f:0, corePoolSize=2, maxPoolSize=2 2024-12-11T03:46:14,430 DEBUG [RS:2;c49966d8a95f:43237 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:14,431 DEBUG [RS:2;c49966d8a95f:43237 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:14,431 DEBUG [RS:2;c49966d8a95f:43237 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:14,431 DEBUG [RS:2;c49966d8a95f:43237 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:14,431 DEBUG [RS:2;c49966d8a95f:43237 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:14,431 DEBUG [RS:2;c49966d8a95f:43237 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c49966d8a95f:0, corePoolSize=1, maxPoolSize=1 2024-12-11T03:46:14,431 DEBUG [RS:2;c49966d8a95f:43237 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c49966d8a95f:0, corePoolSize=3, maxPoolSize=3 2024-12-11T03:46:14,431 DEBUG [RS:2;c49966d8a95f:43237 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c49966d8a95f:0, corePoolSize=3, maxPoolSize=3 2024-12-11T03:46:14,432 INFO [RS:2;c49966d8a95f:43237 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:14,432 INFO [RS:2;c49966d8a95f:43237 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:14,432 INFO [RS:2;c49966d8a95f:43237 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:14,432 INFO [RS:2;c49966d8a95f:43237 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:14,432 INFO [RS:2;c49966d8a95f:43237 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:14,432 INFO [RS:2;c49966d8a95f:43237 {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,43237,1733888774358-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T03:46:14,457 INFO [RS:2;c49966d8a95f:43237 {}] regionserver.HeapMemoryManager(213): Starting, tuneOn=false 2024-12-11T03:46:14,457 INFO [RS:2;c49966d8a95f:43237 {}] hbase.ChoreService(168): Chore ScheduledChore name=c49966d8a95f,43237,1733888774358-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:14,458 INFO [RS:2;c49966d8a95f:43237 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:14,458 INFO [RS:2;c49966d8a95f:43237 {}] regionserver.Replication(171): c49966d8a95f,43237,1733888774358 started 2024-12-11T03:46:14,482 INFO [RS:2;c49966d8a95f:43237 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T03:46:14,482 INFO [RS:2;c49966d8a95f:43237 {}] regionserver.HRegionServer(1482): Serving as c49966d8a95f,43237,1733888774358, RpcServer on c49966d8a95f/172.17.0.2:43237, sessionid=0x1019fa0ee5f0003 2024-12-11T03:46:14,482 DEBUG [RS:2;c49966d8a95f:43237 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-11T03:46:14,482 DEBUG [RS:2;c49966d8a95f:43237 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c49966d8a95f,43237,1733888774358 2024-12-11T03:46:14,482 DEBUG [RS:2;c49966d8a95f:43237 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c49966d8a95f,43237,1733888774358' 2024-12-11T03:46:14,482 DEBUG [RS:2;c49966d8a95f:43237 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-11T03:46:14,483 DEBUG [RS:2;c49966d8a95f:43237 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-11T03:46:14,484 DEBUG [RS:2;c49966d8a95f:43237 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-11T03:46:14,484 DEBUG [RS:2;c49966d8a95f:43237 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-11T03:46:14,484 DEBUG [RS:2;c49966d8a95f:43237 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c49966d8a95f,43237,1733888774358 2024-12-11T03:46:14,484 DEBUG [RS:2;c49966d8a95f:43237 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c49966d8a95f,43237,1733888774358' 2024-12-11T03:46:14,484 DEBUG [RS:2;c49966d8a95f:43237 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-11T03:46:14,485 DEBUG [RS:2;c49966d8a95f:43237 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-11T03:46:14,485 DEBUG [RS:2;c49966d8a95f:43237 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-11T03:46:14,485 INFO [RS:2;c49966d8a95f:43237 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-11T03:46:14,485 INFO [RS:2;c49966d8a95f:43237 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-11T03:46:14,586 INFO [RS:2;c49966d8a95f:43237 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T03:46:14,587 INFO [RS:2;c49966d8a95f:43237 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c49966d8a95f%2C43237%2C1733888774358, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,43237,1733888774358, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/oldWALs, maxLogs=55 2024-12-11T03:46:14,613 INFO [RS:2;c49966d8a95f:43237 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/WALs/c49966d8a95f,43237,1733888774358/c49966d8a95f%2C43237%2C1733888774358.1733888774588 2024-12-11T03:46:14,613 DEBUG [RS:2;c49966d8a95f:43237 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-11T03:46:14,982 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-11T03:46:14,982 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T03:46:14,983 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T03:46:14,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T03:46:14,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T03:46:14,983 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T03:46:14,983 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-11T03:46:14,983 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2114538340, stopped=false 2024-12-11T03:46:14,983 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c49966d8a95f,34767,1733888771039 2024-12-11T03:46:14,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-11T03:46:14,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43237-0x1019fa0ee5f0003, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-11T03:46:14,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-11T03:46:14,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:14,985 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T03:46:14,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:14,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43237-0x1019fa0ee5f0003, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:14,985 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T03:46:14,986 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T03:46:14,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T03:46:14,986 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c49966d8a95f,37977,1733888771090' ***** 2024-12-11T03:46:14,986 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T03:46:14,986 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43237-0x1019fa0ee5f0003, quorum=127.0.0.1:55193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T03:46:14,986 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T03:46:14,986 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-11T03:46:14,986 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c49966d8a95f,43237,1733888774358' ***** 2024-12-11T03:46:14,986 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-11T03:46:14,986 INFO [RS:0;c49966d8a95f:37977 {}] bucket.BucketCache(1738): Disabling cache 2024-12-11T03:46:14,986 INFO [RS:2;c49966d8a95f:43237 {}] bucket.BucketCache(1738): Disabling cache 2024-12-11T03:46:14,986 INFO [RS:0;c49966d8a95f:37977 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284/bucket.persistence 2024-12-11T03:46:14,986 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-11T03:46:14,986 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-11T03:46:14,987 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-11T03:46:14,987 INFO [RS:0;c49966d8a95f:37977 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-11T03:46:14,987 INFO [RS:2;c49966d8a95f:43237 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3bdbdcd7-ea56-c0f2-0322-3a132cf11284/bucket.persistence 2024-12-11T03:46:14,987 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-11T03:46:14,987 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-11T03:46:14,987 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-11T03:46:14,987 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-11T03:46:14,987 INFO [RS:2;c49966d8a95f:43237 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-11T03:46:14,988 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-11T03:46:14,997 DEBUG [RS:0;c49966d8a95f:37977 {}] bucket.BucketCache(1487): Thread RS:0;c49966d8a95f:37977 started persisting bucket cache to file 2024-12-11T03:46:14,997 DEBUG [RS:0;c49966d8a95f:37977 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T03:46:14,997 DEBUG [RS:0;c49966d8a95f:37977 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-11T03:46:14,998 DEBUG [RS:2;c49966d8a95f:43237 {}] bucket.BucketCache(1487): Thread RS:2;c49966d8a95f:43237 started persisting bucket cache to file 2024-12-11T03:46:14,998 DEBUG [RS:2;c49966d8a95f:43237 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T03:46:14,998 DEBUG [RS:2;c49966d8a95f:43237 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-11T03:46:15,001 DEBUG [RS:0;c49966d8a95f:37977 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-11T03:46:15,001 DEBUG [RS:0;c49966d8a95f:37977 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733888774997 2024-12-11T03:46:15,001 DEBUG [RS:0;c49966d8a95f:37977 {}] bucket.BucketCache(1509): Thread RS:0;c49966d8a95f:37977 finished persisting bucket cache to file, renaming 2024-12-11T03:46:15,001 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.HeapMemoryManager(224): Stopping 2024-12-11T03:46:15,001 ERROR [RS:2;c49966d8a95f:43237 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1056) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T03:46:15,001 INFO [RS:0;c49966d8a95f:37977 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-11T03:46:15,001 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-11T03:46:15,001 INFO [RS:0;c49966d8a95f:37977 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-11T03:46:15,001 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.HRegionServer(3091): Received CLOSE for 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:15,001 DEBUG [RS:2;c49966d8a95f:43237 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-11T03:46:15,001 DEBUG [RS:2;c49966d8a95f:43237 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733888774998 2024-12-11T03:46:15,002 DEBUG [RS:2;c49966d8a95f:43237 {}] bucket.BucketCache(1509): Thread RS:2;c49966d8a95f:43237 finished persisting bucket cache to file, renaming 2024-12-11T03:46:15,002 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.HRegionServer(959): stopping server c49966d8a95f,37977,1733888771090 2024-12-11T03:46:15,002 INFO [RS:0;c49966d8a95f:37977 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T03:46:15,002 INFO [RS:2;c49966d8a95f:43237 {}] regionserver.HeapMemoryManager(224): Stopping 2024-12-11T03:46:15,002 INFO [RS:0;c49966d8a95f:37977 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c49966d8a95f:37977. 2024-12-11T03:46:15,002 DEBUG [RS:0;c49966d8a95f:37977 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T03:46:15,002 INFO [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-11T03:46:15,002 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-11T03:46:15,002 INFO [RS:2;c49966d8a95f:43237 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-11T03:46:15,002 DEBUG [RS:0;c49966d8a95f:37977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T03:46:15,002 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 34d44a2e3ad1c784cab546a4453e1557, disabling compactions & flushes 2024-12-11T03:46:15,002 INFO [RS:2;c49966d8a95f:43237 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-11T03:46:15,002 INFO [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. 2024-12-11T03:46:15,002 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-11T03:46:15,002 INFO [RS:2;c49966d8a95f:43237 {}] regionserver.HRegionServer(959): stopping server c49966d8a95f,43237,1733888774358 2024-12-11T03:46:15,002 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. 2024-12-11T03:46:15,002 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-11T03:46:15,002 INFO [RS:2;c49966d8a95f:43237 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T03:46:15,002 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-11T03:46:15,002 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. after waiting 0 ms 2024-12-11T03:46:15,002 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. 2024-12-11T03:46:15,002 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-11T03:46:15,002 INFO [RS:2;c49966d8a95f:43237 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;c49966d8a95f:43237. 2024-12-11T03:46:15,002 DEBUG [RS:2;c49966d8a95f:43237 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T03:46:15,003 DEBUG [RS:2;c49966d8a95f:43237 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T03:46:15,003 INFO [RS:2;c49966d8a95f:43237 {}] regionserver.HRegionServer(976): stopping server c49966d8a95f,43237,1733888774358; all regions closed. 2024-12-11T03:46:15,003 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-11T03:46:15,003 DEBUG [RS:0;c49966d8a95f:37977 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 34d44a2e3ad1c784cab546a4453e1557=testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557.} 2024-12-11T03:46:15,003 DEBUG [RS:0;c49966d8a95f:37977 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 34d44a2e3ad1c784cab546a4453e1557 2024-12-11T03:46:15,003 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-11T03:46:15,003 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-11T03:46:15,003 INFO [regionserver/c49966d8a95f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T03:46:15,003 DEBUG [StoreFileCloser-34d44a2e3ad1c784cab546a4453e1557-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557/family/037b1277e2cf407bb0d03379c2aba103 2024-12-11T03:46:15,003 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-11T03:46:15,003 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-11T03:46:15,003 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-11T03:46:15,003 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-11T03:46:15,004 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-12-11T03:46:15,005 DEBUG [RS:2;c49966d8a95f:43237 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/oldWALs 2024-12-11T03:46:15,005 INFO [RS:2;c49966d8a95f:43237 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c49966d8a95f%2C43237%2C1733888774358:(num 1733888774588) 2024-12-11T03:46:15,005 DEBUG [RS:2;c49966d8a95f:43237 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T03:46:15,005 INFO [RS:2;c49966d8a95f:43237 {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T03:46:15,005 INFO [RS:2;c49966d8a95f:43237 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T03:46:15,005 INFO [RS:2;c49966d8a95f:43237 {}] hbase.ChoreService(370): Chore service for: regionserver/c49966d8a95f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-11T03:46:15,005 INFO [RS:2;c49966d8a95f:43237 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-11T03:46:15,005 INFO [RS:2;c49966d8a95f:43237 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-11T03:46:15,005 INFO [RS:2;c49966d8a95f:43237 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-11T03:46:15,005 INFO [RS:2;c49966d8a95f:43237 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T03:46:15,005 INFO [regionserver/c49966d8a95f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T03:46:15,005 INFO [RS:2;c49966d8a95f:43237 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43237 2024-12-11T03:46:15,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T03:46:15,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43237-0x1019fa0ee5f0003, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c49966d8a95f,43237,1733888774358 2024-12-11T03:46:15,007 INFO [RS:2;c49966d8a95f:43237 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T03:46:15,008 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/default/testBlockEvictionOnGracefulStop1733888772155/34d44a2e3ad1c784cab546a4453e1557/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-11T03:46:15,009 INFO [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. 2024-12-11T03:46:15,009 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 34d44a2e3ad1c784cab546a4453e1557: Waiting for close lock at 1733888775002Running coprocessor pre-close hooks at 1733888775002Disabling compacts and flushes for region at 1733888775002Disabling writes for close at 1733888775002Writing region close event to WAL at 1733888775003 (+1 ms)Running coprocessor post-close hooks at 1733888775009 (+6 ms)Closed at 1733888775009 2024-12-11T03:46:15,009 DEBUG [RS_CLOSE_REGION-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557. 2024-12-11T03:46:15,009 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c49966d8a95f,43237,1733888774358] 2024-12-11T03:46:15,012 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c49966d8a95f,43237,1733888774358 already deleted, retry=false 2024-12-11T03:46:15,012 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c49966d8a95f,43237,1733888774358 expired; onlineServers=1 2024-12-11T03:46:15,016 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9e6bf3c7f53e45a8b7e03df1cdc149df 2024-12-11T03:46:15,030 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/.tmp/info/9e6bf3c7f53e45a8b7e03df1cdc149df is 209, key is testBlockEvictionOnGracefulStop1733888772155,,1733888772160.34d44a2e3ad1c784cab546a4453e1557./info:regioninfo/1733888774412/Put/seqid=0 2024-12-11T03:46:15,031 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/.tmp/info/9e6bf3c7f53e45a8b7e03df1cdc149df 2024-12-11T03:46:15,033 INFO [regionserver/c49966d8a95f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T03:46:15,041 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e80f528ea2fb4938ba7130f9e6ecaeb7 2024-12-11T03:46:15,056 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/.tmp/ns/e80f528ea2fb4938ba7130f9e6ecaeb7 is 43, key is default/ns:d/1733888772031/Put/seqid=0 2024-12-11T03:46:15,056 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/.tmp/ns/e80f528ea2fb4938ba7130f9e6ecaeb7 2024-12-11T03:46:15,066 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6ac110c7e907442e9a493d5be6e34636 2024-12-11T03:46:15,080 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/.tmp/table/6ac110c7e907442e9a493d5be6e34636 is 80, key is testBlockEvictionOnGracefulStop1733888772155/table:state/1733888772531/Put/seqid=0 2024-12-11T03:46:15,080 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/.tmp/table/6ac110c7e907442e9a493d5be6e34636 2024-12-11T03:46:15,082 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/.tmp/info/9e6bf3c7f53e45a8b7e03df1cdc149df as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/info/9e6bf3c7f53e45a8b7e03df1cdc149df 2024-12-11T03:46:15,084 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/info/9e6bf3c7f53e45a8b7e03df1cdc149df, delay=986 ms 2024-12-11T03:46:15,085 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/info/9e6bf3c7f53e45a8b7e03df1cdc149df, entries=14, sequenceid=13, filesize=7.5 K 2024-12-11T03:46:15,085 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/.tmp/ns/e80f528ea2fb4938ba7130f9e6ecaeb7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/ns/e80f528ea2fb4938ba7130f9e6ecaeb7 2024-12-11T03:46:15,086 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/ns/e80f528ea2fb4938ba7130f9e6ecaeb7, delay=949 ms 2024-12-11T03:46:15,087 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/ns/e80f528ea2fb4938ba7130f9e6ecaeb7, entries=2, sequenceid=13, filesize=5.0 K 2024-12-11T03:46:15,087 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/.tmp/table/6ac110c7e907442e9a493d5be6e34636 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/table/6ac110c7e907442e9a493d5be6e34636 2024-12-11T03:46:15,089 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/table/6ac110c7e907442e9a493d5be6e34636, delay=930 ms 2024-12-11T03:46:15,089 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/table/6ac110c7e907442e9a493d5be6e34636, entries=2, sequenceid=13, filesize=5.3 K 2024-12-11T03:46:15,090 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 87ms, sequenceid=13, compaction requested=false 2024-12-11T03:46:15,091 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/info/9e6bf3c7f53e45a8b7e03df1cdc149df 2024-12-11T03:46:15,091 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/ns/e80f528ea2fb4938ba7130f9e6ecaeb7 2024-12-11T03:46:15,092 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/table/6ac110c7e907442e9a493d5be6e34636 2024-12-11T03:46:15,097 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-12-11T03:46:15,097 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-11T03:46:15,097 INFO [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-11T03:46:15,098 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733888775003Running coprocessor pre-close hooks at 1733888775003Disabling compacts and flushes for region at 1733888775003Disabling writes for close at 1733888775003Obtaining lock to block concurrent updates at 1733888775004 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733888775004Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1733888775004Flushing stores of hbase:meta,,1.1588230740 at 1733888775005 (+1 ms)Flushing 1588230740/info: creating writer at 1733888775005Flushing 1588230740/info: appending metadata at 1733888775030 (+25 ms)Flushing 1588230740/info: closing flushed file at 1733888775030Flushing 1588230740/ns: creating writer at 1733888775033 (+3 ms)Flushing 1588230740/ns: appending metadata at 1733888775055 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1733888775055Flushing 1588230740/table: creating writer at 1733888775059 (+4 ms)Flushing 1588230740/table: appending metadata at 1733888775079 (+20 ms)Flushing 1588230740/table: closing flushed file at 1733888775079Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1532543c: reopening flushed file at 1733888775082 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bf711bd: reopening flushed file at 1733888775085 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@358210e2: reopening flushed file at 1733888775087 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 87ms, sequenceid=13, compaction requested=false at 1733888775090 (+3 ms)Writing region close event to WAL at 1733888775092 (+2 ms)Running coprocessor post-close hooks at 1733888775097 (+5 ms)Closed at 1733888775097 2024-12-11T03:46:15,098 DEBUG [RS_CLOSE_META-regionserver/c49966d8a95f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-11T03:46:15,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43237-0x1019fa0ee5f0003, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T03:46:15,109 INFO [RS:2;c49966d8a95f:43237 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-11T03:46:15,109 INFO [RS:2;c49966d8a95f:43237 {}] regionserver.HRegionServer(1031): Exiting; stopping=c49966d8a95f,43237,1733888774358; zookeeper connection closed. 2024-12-11T03:46:15,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43237-0x1019fa0ee5f0003, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T03:46:15,110 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5d8bc444 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5d8bc444 2024-12-11T03:46:15,203 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.HRegionServer(976): stopping server c49966d8a95f,37977,1733888771090; all regions closed. 2024-12-11T03:46:15,205 DEBUG [RS:0;c49966d8a95f:37977 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/oldWALs 2024-12-11T03:46:15,205 INFO [RS:0;c49966d8a95f:37977 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c49966d8a95f%2C37977%2C1733888771090.meta:.meta(num 1733888771984) 2024-12-11T03:46:15,207 DEBUG [RS:0;c49966d8a95f:37977 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/oldWALs 2024-12-11T03:46:15,207 INFO [RS:0;c49966d8a95f:37977 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c49966d8a95f%2C37977%2C1733888771090:(num 1733888771639) 2024-12-11T03:46:15,207 DEBUG [RS:0;c49966d8a95f:37977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T03:46:15,207 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T03:46:15,207 INFO [RS:0;c49966d8a95f:37977 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T03:46:15,207 INFO [RS:0;c49966d8a95f:37977 {}] hbase.ChoreService(370): Chore service for: regionserver/c49966d8a95f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-11T03:46:15,207 INFO [RS:0;c49966d8a95f:37977 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T03:46:15,207 INFO [regionserver/c49966d8a95f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T03:46:15,208 INFO [RS:0;c49966d8a95f:37977 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37977 2024-12-11T03:46:15,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c49966d8a95f,37977,1733888771090 2024-12-11T03:46:15,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T03:46:15,210 INFO [RS:0;c49966d8a95f:37977 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T03:46:15,212 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c49966d8a95f,37977,1733888771090] 2024-12-11T03:46:15,213 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c49966d8a95f,37977,1733888771090 already deleted, retry=false 2024-12-11T03:46:15,213 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c49966d8a95f,37977,1733888771090 expired; onlineServers=0 2024-12-11T03:46:15,213 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c49966d8a95f,34767,1733888771039' ***** 2024-12-11T03:46:15,213 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-11T03:46:15,213 INFO [M:0;c49966d8a95f:34767 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T03:46:15,213 INFO [M:0;c49966d8a95f:34767 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T03:46:15,213 DEBUG [M:0;c49966d8a95f:34767 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-11T03:46:15,214 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-11T03:46:15,214 DEBUG [M:0;c49966d8a95f:34767 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-11T03:46:15,214 DEBUG [master/c49966d8a95f:0:becomeActiveMaster-HFileCleaner.small.0-1733888771359 {}] cleaner.HFileCleaner(306): Exit Thread[master/c49966d8a95f:0:becomeActiveMaster-HFileCleaner.small.0-1733888771359,5,FailOnTimeoutGroup] 2024-12-11T03:46:15,214 DEBUG [master/c49966d8a95f:0:becomeActiveMaster-HFileCleaner.large.0-1733888771358 {}] cleaner.HFileCleaner(306): Exit Thread[master/c49966d8a95f:0:becomeActiveMaster-HFileCleaner.large.0-1733888771358,5,FailOnTimeoutGroup] 2024-12-11T03:46:15,214 INFO [M:0;c49966d8a95f:34767 {}] hbase.ChoreService(370): Chore service for: master/c49966d8a95f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-11T03:46:15,214 INFO [M:0;c49966d8a95f:34767 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T03:46:15,214 DEBUG [M:0;c49966d8a95f:34767 {}] master.HMaster(1795): Stopping service threads 2024-12-11T03:46:15,214 INFO [M:0;c49966d8a95f:34767 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-11T03:46:15,214 INFO [M:0;c49966d8a95f:34767 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T03:46:15,214 INFO [M:0;c49966d8a95f:34767 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T03:46:15,215 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-11T03:46:15,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-11T03:46:15,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T03:46:15,217 DEBUG [M:0;c49966d8a95f:34767 {}] zookeeper.ZKUtil(347): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-11T03:46:15,218 WARN [M:0;c49966d8a95f:34767 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-11T03:46:15,218 INFO [M:0;c49966d8a95f:34767 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/.lastflushedseqids 2024-12-11T03:46:15,225 INFO [M:0;c49966d8a95f:34767 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T03:46:15,225 INFO [M:0;c49966d8a95f:34767 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-11T03:46:15,225 DEBUG [M:0;c49966d8a95f:34767 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-11T03:46:15,225 INFO [M:0;c49966d8a95f:34767 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T03:46:15,225 DEBUG [M:0;c49966d8a95f:34767 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T03:46:15,225 DEBUG [M:0;c49966d8a95f:34767 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-11T03:46:15,225 DEBUG [M:0;c49966d8a95f:34767 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T03:46:15,225 INFO [M:0;c49966d8a95f:34767 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-12-11T03:46:15,236 DEBUG [M:0;c49966d8a95f:34767 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1a9682543a7148b2bf801b8daf9c023d 2024-12-11T03:46:15,250 DEBUG [M:0;c49966d8a95f:34767 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1a9682543a7148b2bf801b8daf9c023d is 82, key is hbase:meta,,1/info:regioninfo/1733888772016/Put/seqid=0 2024-12-11T03:46:15,251 INFO [M:0;c49966d8a95f:34767 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1a9682543a7148b2bf801b8daf9c023d 2024-12-11T03:46:15,260 DEBUG [M:0;c49966d8a95f:34767 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 240e4cb935d74e71b5dec1d24fea48e5 2024-12-11T03:46:15,273 DEBUG [M:0;c49966d8a95f:34767 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/240e4cb935d74e71b5dec1d24fea48e5 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733888772535/Put/seqid=0 2024-12-11T03:46:15,274 INFO [M:0;c49966d8a95f:34767 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/240e4cb935d74e71b5dec1d24fea48e5 2024-12-11T03:46:15,282 DEBUG [M:0;c49966d8a95f:34767 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bcee9e2e9e3a4a3596af1c484b0377f8 2024-12-11T03:46:15,296 DEBUG [M:0;c49966d8a95f:34767 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bcee9e2e9e3a4a3596af1c484b0377f8 is 69, key is c49966d8a95f,37977,1733888771090/rs:state/1733888771466/Put/seqid=0 2024-12-11T03:46:15,297 INFO [M:0;c49966d8a95f:34767 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bcee9e2e9e3a4a3596af1c484b0377f8 2024-12-11T03:46:15,299 INFO [M:0;c49966d8a95f:34767 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bcee9e2e9e3a4a3596af1c484b0377f8 2024-12-11T03:46:15,299 DEBUG [M:0;c49966d8a95f:34767 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1a9682543a7148b2bf801b8daf9c023d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1a9682543a7148b2bf801b8daf9c023d 2024-12-11T03:46:15,301 INFO [M:0;c49966d8a95f:34767 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1a9682543a7148b2bf801b8daf9c023d, entries=8, sequenceid=83, filesize=5.5 K 2024-12-11T03:46:15,301 DEBUG [M:0;c49966d8a95f:34767 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/240e4cb935d74e71b5dec1d24fea48e5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/240e4cb935d74e71b5dec1d24fea48e5 2024-12-11T03:46:15,303 INFO [M:0;c49966d8a95f:34767 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/240e4cb935d74e71b5dec1d24fea48e5, entries=9, sequenceid=83, filesize=6.4 K 2024-12-11T03:46:15,303 DEBUG [M:0;c49966d8a95f:34767 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bcee9e2e9e3a4a3596af1c484b0377f8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bcee9e2e9e3a4a3596af1c484b0377f8 2024-12-11T03:46:15,305 INFO [M:0;c49966d8a95f:34767 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bcee9e2e9e3a4a3596af1c484b0377f8 2024-12-11T03:46:15,305 INFO [M:0;c49966d8a95f:34767 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d834ffd-0e18-abac-0d7a-b3b96ca45511/ac0fbd15-abe8-4b2d-805c-5adb19bfe50e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bcee9e2e9e3a4a3596af1c484b0377f8, entries=3, sequenceid=83, filesize=5.3 K 2024-12-11T03:46:15,306 INFO [M:0;c49966d8a95f:34767 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 81ms, sequenceid=83, compaction requested=false 2024-12-11T03:46:15,308 INFO [M:0;c49966d8a95f:34767 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T03:46:15,308 DEBUG [M:0;c49966d8a95f:34767 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733888775225Disabling compacts and flushes for region at 1733888775225Disabling writes for close at 1733888775225Obtaining lock to block concurrent updates at 1733888775225Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733888775225Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1733888775226 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733888775226Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733888775226Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733888775250 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733888775250Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733888775253 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733888775273 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733888775273Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733888775276 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733888775296 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733888775296Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3999abc8: reopening flushed file at 1733888775299 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ee1a327: reopening flushed file at 1733888775301 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f2127fe: reopening flushed file at 1733888775303 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 81ms, sequenceid=83, compaction requested=false at 1733888775306 (+3 ms)Writing region close event to WAL at 1733888775308 (+2 ms)Closed at 1733888775308 2024-12-11T03:46:15,310 INFO [M:0;c49966d8a95f:34767 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-11T03:46:15,310 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T03:46:15,310 INFO [M:0;c49966d8a95f:34767 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34767 2024-12-11T03:46:15,310 INFO [M:0;c49966d8a95f:34767 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T03:46:15,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T03:46:15,312 INFO [RS:0;c49966d8a95f:37977 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-11T03:46:15,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37977-0x1019fa0ee5f0001, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T03:46:15,312 INFO [RS:0;c49966d8a95f:37977 {}] regionserver.HRegionServer(1031): Exiting; stopping=c49966d8a95f,37977,1733888771090; zookeeper connection closed. 2024-12-11T03:46:15,313 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1269029d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1269029d 2024-12-11T03:46:15,313 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-11T03:46:15,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T03:46:15,412 INFO [M:0;c49966d8a95f:34767 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-11T03:46:15,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34767-0x1019fa0ee5f0000, quorum=127.0.0.1:55193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T03:46:15,415 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-11T03:46:15,416 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-11T03:46:15,417 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T03:46:15,423 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=193 (was 184) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4134 (was 4218)