2024-11-08 00:48:08,132 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-08 00:48:08,144 main DEBUG Took 0.010080 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 00:48:08,145 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 00:48:08,145 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 00:48:08,152 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 00:48:08,153 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:48:08,160 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 00:48:08,172 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:48:08,173 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:48:08,174 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:48:08,174 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:48:08,175 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:48:08,175 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:48:08,176 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:48:08,176 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:48:08,176 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:48:08,177 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:48:08,178 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:48:08,178 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:48:08,179 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:48:08,179 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:48:08,179 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:48:08,180 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:48:08,180 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:48:08,180 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:48:08,181 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:48:08,181 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:48:08,182 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:48:08,182 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:48:08,182 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:48:08,183 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:48:08,183 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:48:08,183 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 00:48:08,185 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:48:08,186 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 00:48:08,187 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 00:48:08,188 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 00:48:08,189 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 00:48:08,189 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 00:48:08,197 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 00:48:08,200 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 00:48:08,202 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 00:48:08,202 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 00:48:08,202 main DEBUG createAppenders(={Console}) 2024-11-08 00:48:08,203 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-08 00:48:08,203 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-08 00:48:08,203 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-08 00:48:08,204 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 00:48:08,205 main DEBUG OutputStream closed 2024-11-08 00:48:08,205 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 00:48:08,205 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 00:48:08,205 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-08 00:48:08,287 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 00:48:08,289 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 00:48:08,290 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 00:48:08,291 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 00:48:08,291 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 00:48:08,292 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 00:48:08,292 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 00:48:08,292 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 00:48:08,292 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 00:48:08,293 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 00:48:08,293 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 00:48:08,293 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 00:48:08,294 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 00:48:08,294 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 00:48:08,294 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 00:48:08,294 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 00:48:08,295 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 00:48:08,295 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 00:48:08,298 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 00:48:08,298 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-08 00:48:08,299 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 00:48:08,299 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-08T00:48:08,312 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-08 00:48:08,315 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 00:48:08,316 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T00:48:08,597 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8 2024-11-08T00:48:08,631 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=829, ProcessCount=11, AvailableMemoryMB=922 2024-11-08T00:48:08,754 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T00:48:08,848 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:08,864 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/cluster_0a91e3ad-13ee-68f1-a12b-45510146a3b1, deleteOnExit=true 2024-11-08T00:48:08,957 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/cluster_0a91e3ad-13ee-68f1-a12b-45510146a3b1/zookeeper_0, clientPort=64158, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/cluster_0a91e3ad-13ee-68f1-a12b-45510146a3b1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/cluster_0a91e3ad-13ee-68f1-a12b-45510146a3b1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T00:48:08,971 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64158 2024-11-08T00:48:08,972 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:09,063 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77 with version=8 2024-11-08T00:48:09,063 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/hbase-staging 2024-11-08T00:48:09,164 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T00:48:09,522 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c302dddf724e:0 server-side Connection retries=45 2024-11-08T00:48:09,533 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:09,535 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:09,544 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T00:48:09,545 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:09,545 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T00:48:09,693 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T00:48:09,788 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T00:48:09,837 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T00:48:09,850 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T00:48:09,855 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T00:48:09,904 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 18005 (auto-detected) 2024-11-08T00:48:09,906 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-08T00:48:09,936 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37429 2024-11-08T00:48:09,960 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37429 connecting to ZooKeeper ensemble=127.0.0.1:64158 2024-11-08T00:48:09,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:374290x0, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T00:48:10,003 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37429-0x100f50c136e0000 connected 2024-11-08T00:48:10,044 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:10,045 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:10,053 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:48:10,059 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77, hbase.cluster.distributed=false 2024-11-08T00:48:10,090 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T00:48:10,101 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37429 2024-11-08T00:48:10,102 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37429 2024-11-08T00:48:10,107 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37429 2024-11-08T00:48:10,109 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37429 2024-11-08T00:48:10,111 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37429 2024-11-08T00:48:10,221 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c302dddf724e:0 server-side Connection retries=45 2024-11-08T00:48:10,223 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:10,223 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:10,223 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T00:48:10,224 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:10,224 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T00:48:10,227 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T00:48:10,230 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T00:48:10,233 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32815 2024-11-08T00:48:10,235 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32815 connecting to ZooKeeper ensemble=127.0.0.1:64158 2024-11-08T00:48:10,235 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:10,237 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:10,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:328150x0, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T00:48:10,249 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32815-0x100f50c136e0001 connected 2024-11-08T00:48:10,249 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:48:10,258 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T00:48:10,277 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/bucket.cache 2024-11-08T00:48:10,285 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T00:48:10,287 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T00:48:10,289 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T00:48:10,292 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T00:48:10,293 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T00:48:10,306 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T00:48:10,306 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/bucket.persistence is complete. 2024-11-08T00:48:10,309 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T00:48:10,317 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T00:48:10,321 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T00:48:10,331 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T00:48:10,340 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32815 2024-11-08T00:48:10,345 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32815 2024-11-08T00:48:10,372 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32815 2024-11-08T00:48:10,392 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32815 2024-11-08T00:48:10,393 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32815 2024-11-08T00:48:10,422 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c302dddf724e:0 server-side Connection retries=45 2024-11-08T00:48:10,423 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:10,423 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:10,424 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T00:48:10,424 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:10,424 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T00:48:10,424 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T00:48:10,425 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T00:48:10,430 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37181 2024-11-08T00:48:10,433 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37181 connecting to ZooKeeper ensemble=127.0.0.1:64158 2024-11-08T00:48:10,434 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:10,435 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:10,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:371810x0, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T00:48:10,449 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:371810x0, quorum=127.0.0.1:64158, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:48:10,450 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T00:48:10,452 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37181-0x100f50c136e0002 connected 2024-11-08T00:48:10,454 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/bucket.cache 2024-11-08T00:48:10,454 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T00:48:10,455 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T00:48:10,455 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T00:48:10,456 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T00:48:10,456 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T00:48:10,456 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T00:48:10,457 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T00:48:10,457 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/bucket.persistence is complete. 2024-11-08T00:48:10,458 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T00:48:10,460 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37181-0x100f50c136e0002, quorum=127.0.0.1:64158, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T00:48:10,464 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37181-0x100f50c136e0002, quorum=127.0.0.1:64158, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T00:48:10,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37181 2024-11-08T00:48:10,472 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37181 2024-11-08T00:48:10,474 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37181 2024-11-08T00:48:10,479 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37181 2024-11-08T00:48:10,479 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37181 2024-11-08T00:48:10,504 DEBUG [M:0;c302dddf724e:37429 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c302dddf724e:37429 2024-11-08T00:48:10,509 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c302dddf724e,37429,1731026889241 2024-11-08T00:48:10,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37181-0x100f50c136e0002, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:48:10,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:48:10,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:48:10,527 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c302dddf724e,37429,1731026889241 2024-11-08T00:48:10,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T00:48:10,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37181-0x100f50c136e0002, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T00:48:10,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:10,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37181-0x100f50c136e0002, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:10,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:10,563 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T00:48:10,566 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c302dddf724e,37429,1731026889241 from backup master directory 2024-11-08T00:48:10,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c302dddf724e,37429,1731026889241 2024-11-08T00:48:10,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37181-0x100f50c136e0002, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:48:10,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:48:10,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:48:10,573 WARN [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T00:48:10,573 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c302dddf724e,37429,1731026889241 2024-11-08T00:48:10,576 INFO [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T00:48:10,581 INFO [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T00:48:10,592 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/hbase.id] with ID: 1b737efd-dbe2-4e1b-8bf4-7a9c015ef7d3 2024-11-08T00:48:10,593 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/.tmp/hbase.id 2024-11-08T00:48:10,602 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/hbase.id] 2024-11-08T00:48:10,681 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:10,688 INFO [master/c302dddf724e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T00:48:10,718 INFO [master/c302dddf724e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 27ms. 2024-11-08T00:48:10,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:10,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:10,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37181-0x100f50c136e0002, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:10,773 INFO [master/c302dddf724e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T00:48:10,777 INFO [master/c302dddf724e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T00:48:10,836 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:48:10,844 INFO [master/c302dddf724e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T00:48:10,939 INFO [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store 2024-11-08T00:48:10,978 INFO [master/c302dddf724e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T00:48:10,983 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:48:10,984 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T00:48:10,985 INFO [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:48:10,985 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:48:10,987 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T00:48:10,987 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:48:10,988 INFO [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:48:10,989 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731026890984Disabling compacts and flushes for region at 1731026890984Disabling writes for close at 1731026890987 (+3 ms)Writing region close event to WAL at 1731026890988 (+1 ms)Closed at 1731026890988 2024-11-08T00:48:10,989 WARN [master/c302dddf724e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/.initializing 2024-11-08T00:48:10,990 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/WALs/c302dddf724e,37429,1731026889241 2024-11-08T00:48:11,013 INFO [master/c302dddf724e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T00:48:11,036 INFO [master/c302dddf724e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c302dddf724e%2C37429%2C1731026889241, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/WALs/c302dddf724e,37429,1731026889241, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/oldWALs, maxLogs=10 2024-11-08T00:48:11,090 INFO [master/c302dddf724e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/WALs/c302dddf724e,37429,1731026889241/c302dddf724e%2C37429%2C1731026889241.1731026891044 2024-11-08T00:48:11,091 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T00:48:11,092 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:48:11,093 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:48:11,098 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:11,099 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:11,146 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:11,189 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T00:48:11,195 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:11,198 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:48:11,199 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:11,214 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T00:48:11,214 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:11,215 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:48:11,216 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:11,229 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T00:48:11,230 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:11,230 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:48:11,231 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:11,237 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T00:48:11,237 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:11,238 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:48:11,238 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:11,243 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:11,244 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:11,249 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:11,250 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:11,253 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T00:48:11,257 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:11,270 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:48:11,272 INFO [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59592415, jitterRate=-0.11200381815433502}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T00:48:11,280 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731026891119Initializing all the Stores at 1731026891120 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026891121 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026891122 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026891123 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026891123Cleaning up temporary data from old regions at 1731026891250 (+127 ms)Region opened successfully at 1731026891280 (+30 ms) 2024-11-08T00:48:11,282 INFO [master/c302dddf724e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T00:48:11,328 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63b9b982, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c302dddf724e/172.17.0.2:0 2024-11-08T00:48:11,384 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T00:48:11,401 INFO [master/c302dddf724e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T00:48:11,402 INFO [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T00:48:11,406 INFO [master/c302dddf724e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:48:11,407 INFO [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:48:11,415 INFO [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-08T00:48:11,416 INFO [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T00:48:11,469 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T00:48:11,480 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T00:48:11,483 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T00:48:11,486 INFO [master/c302dddf724e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T00:48:11,489 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T00:48:11,492 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T00:48:11,495 INFO [master/c302dddf724e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T00:48:11,502 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T00:48:11,505 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T00:48:11,507 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T00:48:11,509 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T00:48:11,531 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T00:48:11,533 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T00:48:11,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T00:48:11,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T00:48:11,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:11,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:11,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37181-0x100f50c136e0002, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T00:48:11,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37181-0x100f50c136e0002, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:11,556 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c302dddf724e,37429,1731026889241, sessionid=0x100f50c136e0000, setting cluster-up flag (Was=false) 2024-11-08T00:48:11,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:11,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37181-0x100f50c136e0002, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:11,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:11,584 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T00:48:11,586 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c302dddf724e,37429,1731026889241 2024-11-08T00:48:11,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37181-0x100f50c136e0002, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:11,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:11,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:11,599 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T00:48:11,601 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c302dddf724e,37429,1731026889241 2024-11-08T00:48:11,606 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T00:48:11,685 INFO [RS:1;c302dddf724e:37181 {}] regionserver.HRegionServer(746): ClusterId : 1b737efd-dbe2-4e1b-8bf4-7a9c015ef7d3 2024-11-08T00:48:11,688 DEBUG [RS:1;c302dddf724e:37181 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T00:48:11,689 INFO [RS:0;c302dddf724e:32815 {}] regionserver.HRegionServer(746): ClusterId : 1b737efd-dbe2-4e1b-8bf4-7a9c015ef7d3 2024-11-08T00:48:11,689 DEBUG [RS:0;c302dddf724e:32815 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T00:48:11,693 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:48:11,695 DEBUG [RS:0;c302dddf724e:32815 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T00:48:11,695 DEBUG [RS:1;c302dddf724e:37181 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T00:48:11,695 DEBUG [RS:0;c302dddf724e:32815 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T00:48:11,695 DEBUG [RS:1;c302dddf724e:37181 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T00:48:11,700 DEBUG [RS:0;c302dddf724e:32815 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T00:48:11,701 DEBUG [RS:1;c302dddf724e:37181 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T00:48:11,701 DEBUG [RS:0;c302dddf724e:32815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14bf6ff6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c302dddf724e/172.17.0.2:0 2024-11-08T00:48:11,701 DEBUG [RS:1;c302dddf724e:37181 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70b3cff5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c302dddf724e/172.17.0.2:0 2024-11-08T00:48:11,707 INFO [master/c302dddf724e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T00:48:11,716 INFO [master/c302dddf724e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T00:48:11,721 DEBUG [RS:1;c302dddf724e:37181 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;c302dddf724e:37181 2024-11-08T00:48:11,722 DEBUG [RS:0;c302dddf724e:32815 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c302dddf724e:32815 2024-11-08T00:48:11,726 INFO [RS:0;c302dddf724e:32815 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T00:48:11,726 INFO [RS:0;c302dddf724e:32815 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T00:48:11,726 DEBUG [RS:0;c302dddf724e:32815 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T00:48:11,723 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c302dddf724e,37429,1731026889241 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T00:48:11,726 INFO [RS:1;c302dddf724e:37181 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T00:48:11,727 INFO [RS:1;c302dddf724e:37181 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T00:48:11,727 DEBUG [RS:1;c302dddf724e:37181 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T00:48:11,729 INFO [RS:0;c302dddf724e:32815 {}] regionserver.HRegionServer(2659): reportForDuty to master=c302dddf724e,37429,1731026889241 with port=32815, startcode=1731026890175 2024-11-08T00:48:11,729 INFO [RS:1;c302dddf724e:37181 {}] regionserver.HRegionServer(2659): reportForDuty to master=c302dddf724e,37429,1731026889241 with port=37181, startcode=1731026890422 2024-11-08T00:48:11,732 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c302dddf724e:0, corePoolSize=5, maxPoolSize=5 2024-11-08T00:48:11,732 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c302dddf724e:0, corePoolSize=5, maxPoolSize=5 2024-11-08T00:48:11,733 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c302dddf724e:0, corePoolSize=5, maxPoolSize=5 2024-11-08T00:48:11,733 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c302dddf724e:0, corePoolSize=5, maxPoolSize=5 2024-11-08T00:48:11,733 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c302dddf724e:0, corePoolSize=10, maxPoolSize=10 2024-11-08T00:48:11,734 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:11,734 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c302dddf724e:0, corePoolSize=2, maxPoolSize=2 2024-11-08T00:48:11,734 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:11,742 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T00:48:11,743 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T00:48:11,744 INFO [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026921744 2024-11-08T00:48:11,746 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T00:48:11,746 DEBUG [RS:0;c302dddf724e:32815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T00:48:11,746 DEBUG [RS:1;c302dddf724e:37181 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T00:48:11,747 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T00:48:11,748 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:11,748 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T00:48:11,752 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T00:48:11,752 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T00:48:11,753 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T00:48:11,753 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T00:48:11,762 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:11,773 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T00:48:11,774 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T00:48:11,775 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T00:48:11,806 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T00:48:11,807 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T00:48:11,809 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c302dddf724e:0:becomeActiveMaster-HFileCleaner.large.0-1731026891808,5,FailOnTimeoutGroup] 2024-11-08T00:48:11,813 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c302dddf724e:0:becomeActiveMaster-HFileCleaner.small.0-1731026891809,5,FailOnTimeoutGroup] 2024-11-08T00:48:11,813 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:11,813 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T00:48:11,815 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:11,815 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:11,834 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T00:48:11,834 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77 2024-11-08T00:48:11,854 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:48:11,856 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T00:48:11,861 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46125, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T00:48:11,861 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41753, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T00:48:11,870 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37429 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c302dddf724e,32815,1731026890175 2024-11-08T00:48:11,873 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37429 {}] master.ServerManager(517): Registering regionserver=c302dddf724e,32815,1731026890175 2024-11-08T00:48:11,874 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T00:48:11,874 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:11,874 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:48:11,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T00:48:11,881 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T00:48:11,881 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:11,881 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:48:11,881 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T00:48:11,889 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37429 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c302dddf724e,37181,1731026890422 2024-11-08T00:48:11,890 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37429 {}] master.ServerManager(517): Registering regionserver=c302dddf724e,37181,1731026890422 2024-11-08T00:48:11,892 DEBUG [RS:0;c302dddf724e:32815 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77 2024-11-08T00:48:11,892 DEBUG [RS:0;c302dddf724e:32815 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T00:48:11,892 DEBUG [RS:0;c302dddf724e:32815 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T00:48:11,894 DEBUG [RS:1;c302dddf724e:37181 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77 2024-11-08T00:48:11,894 DEBUG [RS:1;c302dddf724e:37181 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T00:48:11,894 DEBUG [RS:1;c302dddf724e:37181 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T00:48:11,901 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T00:48:11,902 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:11,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:48:11,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T00:48:11,904 DEBUG [RS:1;c302dddf724e:37181 {}] zookeeper.ZKUtil(111): regionserver:37181-0x100f50c136e0002, quorum=127.0.0.1:64158, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c302dddf724e,37181,1731026890422 2024-11-08T00:48:11,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T00:48:11,905 DEBUG [RS:0;c302dddf724e:32815 {}] zookeeper.ZKUtil(111): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c302dddf724e,32815,1731026890175 2024-11-08T00:48:11,905 WARN [RS:1;c302dddf724e:37181 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T00:48:11,905 WARN [RS:0;c302dddf724e:32815 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T00:48:11,905 INFO [RS:1;c302dddf724e:37181 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T00:48:11,905 INFO [RS:0;c302dddf724e:32815 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T00:48:11,905 DEBUG [RS:1;c302dddf724e:37181 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,37181,1731026890422 2024-11-08T00:48:11,905 DEBUG [RS:0;c302dddf724e:32815 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,32815,1731026890175 2024-11-08T00:48:11,911 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c302dddf724e,32815,1731026890175] 2024-11-08T00:48:11,912 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c302dddf724e,37181,1731026890422] 2024-11-08T00:48:11,921 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T00:48:11,921 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:11,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:48:11,922 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T00:48:11,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740 2024-11-08T00:48:11,924 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740 2024-11-08T00:48:11,924 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T00:48:11,924 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T00:48:11,925 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T00:48:11,926 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T00:48:11,936 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:48:11,936 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69026931, jitterRate=0.028581425547599792}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T00:48:11,941 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731026891855Initializing all the Stores at 1731026891855Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026891855Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026891856 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026891856Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026891856Cleaning up temporary data from old regions at 1731026891924 (+68 ms)Region opened successfully at 1731026891941 (+17 ms) 2024-11-08T00:48:11,941 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T00:48:11,941 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T00:48:11,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T00:48:11,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T00:48:11,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T00:48:11,943 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T00:48:11,944 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731026891941Disabling compacts and flushes for region at 1731026891941Disabling writes for close at 1731026891942 (+1 ms)Writing region close event to WAL at 1731026891943 (+1 ms)Closed at 1731026891943 2024-11-08T00:48:11,946 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T00:48:11,946 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T00:48:11,951 INFO [RS:0;c302dddf724e:32815 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T00:48:11,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T00:48:11,953 INFO [RS:1;c302dddf724e:37181 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T00:48:11,980 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:48:11,982 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T00:48:11,983 INFO [RS:1;c302dddf724e:37181 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T00:48:11,984 INFO [RS:0;c302dddf724e:32815 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T00:48:11,989 INFO [RS:1;c302dddf724e:37181 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T00:48:11,989 INFO [RS:0;c302dddf724e:32815 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T00:48:11,989 INFO [RS:1;c302dddf724e:37181 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:11,989 INFO [RS:0;c302dddf724e:32815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:11,990 INFO [RS:1;c302dddf724e:37181 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T00:48:11,992 INFO [RS:0;c302dddf724e:32815 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T00:48:11,995 INFO [RS:0;c302dddf724e:32815 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T00:48:11,996 INFO [RS:0;c302dddf724e:32815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:11,996 DEBUG [RS:0;c302dddf724e:32815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:11,997 DEBUG [RS:0;c302dddf724e:32815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:11,997 DEBUG [RS:0;c302dddf724e:32815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:11,997 DEBUG [RS:0;c302dddf724e:32815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:11,997 DEBUG [RS:0;c302dddf724e:32815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:11,997 DEBUG [RS:0;c302dddf724e:32815 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c302dddf724e:0, corePoolSize=2, maxPoolSize=2 2024-11-08T00:48:11,997 DEBUG [RS:0;c302dddf724e:32815 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:11,998 DEBUG [RS:0;c302dddf724e:32815 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:11,998 DEBUG [RS:0;c302dddf724e:32815 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:11,998 DEBUG [RS:0;c302dddf724e:32815 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:11,998 DEBUG [RS:0;c302dddf724e:32815 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:11,998 DEBUG [RS:0;c302dddf724e:32815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:11,998 DEBUG [RS:0;c302dddf724e:32815 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c302dddf724e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T00:48:11,998 DEBUG [RS:0;c302dddf724e:32815 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c302dddf724e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T00:48:11,999 INFO [RS:1;c302dddf724e:37181 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T00:48:11,999 INFO [RS:1;c302dddf724e:37181 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:11,999 DEBUG [RS:1;c302dddf724e:37181 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:12,000 DEBUG [RS:1;c302dddf724e:37181 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:12,000 DEBUG [RS:1;c302dddf724e:37181 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:12,000 DEBUG [RS:1;c302dddf724e:37181 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:12,000 DEBUG [RS:1;c302dddf724e:37181 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:12,000 DEBUG [RS:1;c302dddf724e:37181 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c302dddf724e:0, corePoolSize=2, maxPoolSize=2 2024-11-08T00:48:12,001 DEBUG [RS:1;c302dddf724e:37181 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:12,001 DEBUG [RS:1;c302dddf724e:37181 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:12,001 DEBUG [RS:1;c302dddf724e:37181 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:12,001 DEBUG [RS:1;c302dddf724e:37181 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:12,002 DEBUG [RS:1;c302dddf724e:37181 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:12,002 DEBUG [RS:1;c302dddf724e:37181 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:12,002 DEBUG [RS:1;c302dddf724e:37181 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c302dddf724e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T00:48:12,002 DEBUG [RS:1;c302dddf724e:37181 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c302dddf724e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T00:48:12,008 INFO [RS:0;c302dddf724e:32815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,009 INFO [RS:0;c302dddf724e:32815 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,009 INFO [RS:0;c302dddf724e:32815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,009 INFO [RS:0;c302dddf724e:32815 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,009 INFO [RS:0;c302dddf724e:32815 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,009 INFO [RS:0;c302dddf724e:32815 {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,32815,1731026890175-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T00:48:12,012 INFO [RS:1;c302dddf724e:37181 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,012 INFO [RS:1;c302dddf724e:37181 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,013 INFO [RS:1;c302dddf724e:37181 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,013 INFO [RS:1;c302dddf724e:37181 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,013 INFO [RS:1;c302dddf724e:37181 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,013 INFO [RS:1;c302dddf724e:37181 {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,37181,1731026890422-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T00:48:12,042 INFO [RS:0;c302dddf724e:32815 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T00:48:12,042 INFO [RS:1;c302dddf724e:37181 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T00:48:12,044 INFO [RS:0;c302dddf724e:32815 {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,32815,1731026890175-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,044 INFO [RS:1;c302dddf724e:37181 {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,37181,1731026890422-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,045 INFO [RS:1;c302dddf724e:37181 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,045 INFO [RS:0;c302dddf724e:32815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,045 INFO [RS:1;c302dddf724e:37181 {}] regionserver.Replication(171): c302dddf724e,37181,1731026890422 started 2024-11-08T00:48:12,046 INFO [RS:0;c302dddf724e:32815 {}] regionserver.Replication(171): c302dddf724e,32815,1731026890175 started 2024-11-08T00:48:12,076 INFO [RS:0;c302dddf724e:32815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,077 INFO [RS:0;c302dddf724e:32815 {}] regionserver.HRegionServer(1482): Serving as c302dddf724e,32815,1731026890175, RpcServer on c302dddf724e/172.17.0.2:32815, sessionid=0x100f50c136e0001 2024-11-08T00:48:12,078 DEBUG [RS:0;c302dddf724e:32815 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T00:48:12,078 DEBUG [RS:0;c302dddf724e:32815 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c302dddf724e,32815,1731026890175 2024-11-08T00:48:12,078 DEBUG [RS:0;c302dddf724e:32815 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c302dddf724e,32815,1731026890175' 2024-11-08T00:48:12,078 DEBUG [RS:0;c302dddf724e:32815 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T00:48:12,080 DEBUG [RS:0;c302dddf724e:32815 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T00:48:12,081 DEBUG [RS:0;c302dddf724e:32815 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T00:48:12,081 DEBUG [RS:0;c302dddf724e:32815 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T00:48:12,081 DEBUG [RS:0;c302dddf724e:32815 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c302dddf724e,32815,1731026890175 2024-11-08T00:48:12,081 DEBUG [RS:0;c302dddf724e:32815 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c302dddf724e,32815,1731026890175' 2024-11-08T00:48:12,082 DEBUG [RS:0;c302dddf724e:32815 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T00:48:12,083 DEBUG [RS:0;c302dddf724e:32815 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T00:48:12,083 DEBUG [RS:0;c302dddf724e:32815 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T00:48:12,084 INFO [RS:0;c302dddf724e:32815 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T00:48:12,084 INFO [RS:0;c302dddf724e:32815 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T00:48:12,087 INFO [RS:1;c302dddf724e:37181 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,087 INFO [RS:1;c302dddf724e:37181 {}] regionserver.HRegionServer(1482): Serving as c302dddf724e,37181,1731026890422, RpcServer on c302dddf724e/172.17.0.2:37181, sessionid=0x100f50c136e0002 2024-11-08T00:48:12,087 DEBUG [RS:1;c302dddf724e:37181 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T00:48:12,087 DEBUG [RS:1;c302dddf724e:37181 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c302dddf724e,37181,1731026890422 2024-11-08T00:48:12,087 DEBUG [RS:1;c302dddf724e:37181 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c302dddf724e,37181,1731026890422' 2024-11-08T00:48:12,087 DEBUG [RS:1;c302dddf724e:37181 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T00:48:12,088 DEBUG [RS:1;c302dddf724e:37181 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T00:48:12,089 DEBUG [RS:1;c302dddf724e:37181 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T00:48:12,089 DEBUG [RS:1;c302dddf724e:37181 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T00:48:12,089 DEBUG [RS:1;c302dddf724e:37181 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c302dddf724e,37181,1731026890422 2024-11-08T00:48:12,089 DEBUG [RS:1;c302dddf724e:37181 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c302dddf724e,37181,1731026890422' 2024-11-08T00:48:12,089 DEBUG [RS:1;c302dddf724e:37181 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T00:48:12,090 DEBUG [RS:1;c302dddf724e:37181 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T00:48:12,090 DEBUG [RS:1;c302dddf724e:37181 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T00:48:12,091 INFO [RS:1;c302dddf724e:37181 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T00:48:12,091 INFO [RS:1;c302dddf724e:37181 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T00:48:12,134 WARN [c302dddf724e:37429 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T00:48:12,191 INFO [RS:0;c302dddf724e:32815 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T00:48:12,192 INFO [RS:0;c302dddf724e:32815 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c302dddf724e%2C32815%2C1731026890175, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,32815,1731026890175, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/oldWALs, maxLogs=55 2024-11-08T00:48:12,192 INFO [RS:1;c302dddf724e:37181 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T00:48:12,193 INFO [RS:1;c302dddf724e:37181 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c302dddf724e%2C37181%2C1731026890422, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,37181,1731026890422, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/oldWALs, maxLogs=55 2024-11-08T00:48:12,221 INFO [RS:1;c302dddf724e:37181 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,37181,1731026890422/c302dddf724e%2C37181%2C1731026890422.1731026892196 2024-11-08T00:48:12,222 INFO [RS:0;c302dddf724e:32815 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,32815,1731026890175/c302dddf724e%2C32815%2C1731026890175.1731026892194 2024-11-08T00:48:12,222 DEBUG [RS:1;c302dddf724e:37181 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T00:48:12,224 DEBUG [RS:0;c302dddf724e:32815 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T00:48:12,387 DEBUG [c302dddf724e:37429 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-08T00:48:12,403 DEBUG [c302dddf724e:37429 {}] balancer.BalancerClusterState(204): Hosts are {c302dddf724e=0} racks are {/default-rack=0} 2024-11-08T00:48:12,408 DEBUG [c302dddf724e:37429 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T00:48:12,408 DEBUG [c302dddf724e:37429 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T00:48:12,408 DEBUG [c302dddf724e:37429 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T00:48:12,408 DEBUG [c302dddf724e:37429 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T00:48:12,408 INFO [c302dddf724e:37429 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T00:48:12,408 INFO [c302dddf724e:37429 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T00:48:12,408 DEBUG [c302dddf724e:37429 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T00:48:12,416 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c302dddf724e,37181,1731026890422 2024-11-08T00:48:12,422 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,37181,1731026890422, state=OPENING 2024-11-08T00:48:12,428 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T00:48:12,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37181-0x100f50c136e0002, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:12,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:12,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:12,431 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:48:12,431 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:48:12,432 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:48:12,433 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:48:12,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c302dddf724e,37181,1731026890422}] 2024-11-08T00:48:12,612 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T00:48:12,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57655, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T00:48:12,625 INFO [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T00:48:12,625 INFO [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T00:48:12,626 INFO [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T00:48:12,627 INFO [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c302dddf724e%2C37181%2C1731026890422.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,37181,1731026890422, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/oldWALs, maxLogs=55 2024-11-08T00:48:12,656 INFO [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,37181,1731026890422/c302dddf724e%2C37181%2C1731026890422.meta.1731026892629.meta 2024-11-08T00:48:12,657 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T00:48:12,658 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:48:12,659 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T00:48:12,662 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T00:48:12,669 INFO [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T00:48:12,674 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T00:48:12,675 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:48:12,676 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T00:48:12,676 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T00:48:12,678 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T00:48:12,679 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T00:48:12,680 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:12,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:48:12,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T00:48:12,681 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T00:48:12,681 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:12,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:48:12,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T00:48:12,683 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T00:48:12,683 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:12,683 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:48:12,683 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T00:48:12,684 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T00:48:12,684 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:12,684 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:48:12,684 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T00:48:12,685 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740 2024-11-08T00:48:12,686 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740 2024-11-08T00:48:12,687 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T00:48:12,687 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T00:48:12,687 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T00:48:12,689 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T00:48:12,690 INFO [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67975728, jitterRate=0.012917280197143555}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T00:48:12,690 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T00:48:12,692 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731026892677Writing region info on filesystem at 1731026892677Initializing all the Stores at 1731026892677Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026892677Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026892678 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026892678Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026892678Cleaning up temporary data from old regions at 1731026892687 (+9 ms)Running coprocessor post-open hooks at 1731026892690 (+3 ms)Region opened successfully at 1731026892692 (+2 ms) 2024-11-08T00:48:12,699 INFO [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731026892603 2024-11-08T00:48:12,713 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T00:48:12,713 INFO [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T00:48:12,714 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c302dddf724e,37181,1731026890422 2024-11-08T00:48:12,716 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,37181,1731026890422, state=OPEN 2024-11-08T00:48:12,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37181-0x100f50c136e0002, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:48:12,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:48:12,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:48:12,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:48:12,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:48:12,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:48:12,720 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c302dddf724e,37181,1731026890422 2024-11-08T00:48:12,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T00:48:12,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c302dddf724e,37181,1731026890422 in 286 msec 2024-11-08T00:48:12,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T00:48:12,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 771 msec 2024-11-08T00:48:12,730 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T00:48:12,730 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T00:48:12,755 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:48:12,756 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c302dddf724e,37181,1731026890422, seqNum=-1] 2024-11-08T00:48:12,778 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:48:12,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32775, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:48:12,800 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1560 sec 2024-11-08T00:48:12,800 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731026892800, completionTime=-1 2024-11-08T00:48:12,803 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-08T00:48:12,803 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T00:48:12,831 INFO [master/c302dddf724e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-08T00:48:12,831 INFO [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026952831 2024-11-08T00:48:12,831 INFO [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731027012831 2024-11-08T00:48:12,831 INFO [master/c302dddf724e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-08T00:48:12,833 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:48:12,839 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,37429,1731026889241-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,840 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,37429,1731026889241-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,840 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,37429,1731026889241-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,841 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c302dddf724e:37429, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,841 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,842 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,849 DEBUG [master/c302dddf724e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T00:48:12,870 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.296sec 2024-11-08T00:48:12,871 INFO [master/c302dddf724e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T00:48:12,873 INFO [master/c302dddf724e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T00:48:12,874 INFO [master/c302dddf724e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T00:48:12,874 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T00:48:12,874 INFO [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T00:48:12,875 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,37429,1731026889241-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T00:48:12,875 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,37429,1731026889241-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T00:48:12,877 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T00:48:12,878 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T00:48:12,878 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,37429,1731026889241-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:12,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32be4008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:48:12,896 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T00:48:12,896 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T00:48:12,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c302dddf724e,37429,-1 for getting cluster id 2024-11-08T00:48:12,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:48:12,911 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1b737efd-dbe2-4e1b-8bf4-7a9c015ef7d3' 2024-11-08T00:48:12,914 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:48:12,915 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1b737efd-dbe2-4e1b-8bf4-7a9c015ef7d3" 2024-11-08T00:48:12,917 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@312cf260, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:48:12,917 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c302dddf724e,37429,-1] 2024-11-08T00:48:12,920 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:48:12,922 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:48:12,925 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:48:12,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1061b362, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:48:12,929 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:48:12,937 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c302dddf724e,37181,1731026890422, seqNum=-1] 2024-11-08T00:48:12,937 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:48:12,941 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:48:12,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c302dddf724e,37429,1731026889241 2024-11-08T00:48:12,993 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T00:48:12,998 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c302dddf724e,37429,1731026889241 2024-11-08T00:48:13,002 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1021df04 2024-11-08T00:48:13,003 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T00:48:13,006 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T00:48:13,014 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37429 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731026892989', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T00:48:13,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37429 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731026892989 2024-11-08T00:48:13,026 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731026892989 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T00:48:13,028 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37429 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731026892989" procId is: 4 2024-11-08T00:48:13,029 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:13,031 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731026892989 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T00:48:13,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T00:48:13,054 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731026892989-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1cbc8c34c2142cc68f3b38bf0794ae6e, NAME => 'testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731026892989', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77 2024-11-08T00:48:13,062 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731026892989-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:48:13,062 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731026892989-pool-0 {}] regionserver.HRegion(1722): Closing 1cbc8c34c2142cc68f3b38bf0794ae6e, disabling compactions & flushes 2024-11-08T00:48:13,062 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731026892989-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:13,062 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731026892989-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:13,062 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731026892989-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. after waiting 0 ms 2024-11-08T00:48:13,062 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731026892989-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:13,063 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731026892989-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:13,063 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731026892989-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1cbc8c34c2142cc68f3b38bf0794ae6e: Waiting for close lock at 1731026893062Disabling compacts and flushes for region at 1731026893062Disabling writes for close at 1731026893062Writing region close event to WAL at 1731026893063 (+1 ms)Closed at 1731026893063 2024-11-08T00:48:13,065 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731026892989 execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T00:48:13,071 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731026893065"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731026893065"}]},"ts":"1731026893065"} 2024-11-08T00:48:13,078 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T00:48:13,079 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731026892989 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T00:48:13,082 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731026892989","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731026893079"}]},"ts":"1731026893079"} 2024-11-08T00:48:13,086 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731026892989, state=ENABLING in hbase:meta 2024-11-08T00:48:13,087 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {c302dddf724e=0} racks are {/default-rack=0} 2024-11-08T00:48:13,087 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T00:48:13,087 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T00:48:13,087 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T00:48:13,087 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T00:48:13,087 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T00:48:13,088 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T00:48:13,088 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T00:48:13,089 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731026892989, region=1cbc8c34c2142cc68f3b38bf0794ae6e, ASSIGN}] 2024-11-08T00:48:13,090 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731026892989, region=1cbc8c34c2142cc68f3b38bf0794ae6e, ASSIGN 2024-11-08T00:48:13,091 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731026892989, region=1cbc8c34c2142cc68f3b38bf0794ae6e, ASSIGN; state=OFFLINE, location=c302dddf724e,37181,1731026890422; forceNewPlan=false, retain=false 2024-11-08T00:48:13,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T00:48:13,244 INFO [c302dddf724e:37429 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-08T00:48:13,245 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1cbc8c34c2142cc68f3b38bf0794ae6e, regionState=OPENING, regionLocation=c302dddf724e,37181,1731026890422 2024-11-08T00:48:13,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731026892989, region=1cbc8c34c2142cc68f3b38bf0794ae6e, ASSIGN because future has completed 2024-11-08T00:48:13,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cbc8c34c2142cc68f3b38bf0794ae6e, server=c302dddf724e,37181,1731026890422}] 2024-11-08T00:48:13,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T00:48:13,406 INFO [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:13,406 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1cbc8c34c2142cc68f3b38bf0794ae6e, NAME => 'testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e.', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:48:13,407 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731026892989 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:13,407 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:48:13,408 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:13,408 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:13,409 INFO [StoreOpener-1cbc8c34c2142cc68f3b38bf0794ae6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:13,414 INFO [StoreOpener-1cbc8c34c2142cc68f3b38bf0794ae6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1cbc8c34c2142cc68f3b38bf0794ae6e columnFamilyName family 2024-11-08T00:48:13,414 DEBUG [StoreOpener-1cbc8c34c2142cc68f3b38bf0794ae6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:13,415 INFO [StoreOpener-1cbc8c34c2142cc68f3b38bf0794ae6e-1 {}] regionserver.HStore(327): Store=1cbc8c34c2142cc68f3b38bf0794ae6e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:48:13,415 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:13,416 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:13,417 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:13,417 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:13,417 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:13,418 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:13,427 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:48:13,428 INFO [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1cbc8c34c2142cc68f3b38bf0794ae6e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63155280, jitterRate=-0.05891299247741699}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:48:13,428 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:13,429 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1cbc8c34c2142cc68f3b38bf0794ae6e: Running coprocessor pre-open hook at 1731026893408Writing region info on filesystem at 1731026893408Initializing all the Stores at 1731026893408Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026893408Cleaning up temporary data from old regions at 1731026893417 (+9 ms)Running coprocessor post-open hooks at 1731026893428 (+11 ms)Region opened successfully at 1731026893429 (+1 ms) 2024-11-08T00:48:13,431 INFO [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e., pid=6, masterSystemTime=1731026893403 2024-11-08T00:48:13,435 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:13,435 INFO [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:13,435 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1cbc8c34c2142cc68f3b38bf0794ae6e, regionState=OPEN, openSeqNum=2, regionLocation=c302dddf724e,37181,1731026890422 2024-11-08T00:48:13,440 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cbc8c34c2142cc68f3b38bf0794ae6e, server=c302dddf724e,37181,1731026890422 because future has completed 2024-11-08T00:48:13,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T00:48:13,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1cbc8c34c2142cc68f3b38bf0794ae6e, server=c302dddf724e,37181,1731026890422 in 191 msec 2024-11-08T00:48:13,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T00:48:13,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731026892989, region=1cbc8c34c2142cc68f3b38bf0794ae6e, ASSIGN in 354 msec 2024-11-08T00:48:13,447 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731026892989 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T00:48:13,448 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731026892989","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731026893447"}]},"ts":"1731026893447"} 2024-11-08T00:48:13,451 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731026892989, state=ENABLED in hbase:meta 2024-11-08T00:48:13,453 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731026892989 execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T00:48:13,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731026892989 in 436 msec 2024-11-08T00:48:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T00:48:13,665 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731026892989 get assigned. Timeout = 60000ms 2024-11-08T00:48:13,665 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731026892989 completed 2024-11-08T00:48:13,667 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T00:48:13,673 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731026892989 assigned to meta. Checking AM states. 2024-11-08T00:48:13,674 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T00:48:13,674 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731026892989 assigned. 2024-11-08T00:48:13,682 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731026892989', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e., hostname=c302dddf724e,37181,1731026890422, seqNum=2] 2024-11-08T00:48:13,694 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1cbc8c34c2142cc68f3b38bf0794ae6e 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-08T00:48:13,729 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2c449b08bd9d43c19af37b0bdec7ee3f 2024-11-08T00:48:13,773 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e/.tmp/family/2c449b08bd9d43c19af37b0bdec7ee3f is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-08T00:48:13,779 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e/.tmp/family/2c449b08bd9d43c19af37b0bdec7ee3f 2024-11-08T00:48:13,819 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e/.tmp/family/2c449b08bd9d43c19af37b0bdec7ee3f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e/family/2c449b08bd9d43c19af37b0bdec7ee3f 2024-11-08T00:48:13,821 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e/family/2c449b08bd9d43c19af37b0bdec7ee3f, delay=927 ms 2024-11-08T00:48:13,823 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e/family/2c449b08bd9d43c19af37b0bdec7ee3f, entries=2, sequenceid=6, filesize=4.9 K 2024-11-08T00:48:13,830 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 1cbc8c34c2142cc68f3b38bf0794ae6e in 135ms, sequenceid=6, compaction requested=false 2024-11-08T00:48:13,832 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731026892989' 2024-11-08T00:48:13,834 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1cbc8c34c2142cc68f3b38bf0794ae6e: 2024-11-08T00:48:14,751 DEBUG [hfile-prefetch-1731026893821 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 2c449b08bd9d43c19af37b0bdec7ee3f_0. 2024-11-08T00:48:14,757 DEBUG [hfile-prefetch-1731026893821 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 2c449b08bd9d43c19af37b0bdec7ee3f_117. 2024-11-08T00:48:14,758 DEBUG [hfile-prefetch-1731026893821 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e/family/2c449b08bd9d43c19af37b0bdec7ee3f, with total blocks 2, and data blocks 1 2024-11-08T00:48:14,758 DEBUG [hfile-prefetch-1731026893821 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-08T00:48:14,758 DEBUG [hfile-prefetch-1731026893821 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e/family/2c449b08bd9d43c19af37b0bdec7ee3f, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-08T00:48:14,759 DEBUG [hfile-prefetch-1731026893821 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e/family/2c449b08bd9d43c19af37b0bdec7ee3f. Will sleep 100ms and try the verification again. 2024-11-08T00:48:14,766 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-08T00:48:14,766 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-08T00:48:14,767 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T00:48:14,767 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T00:48:14,793 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-08T00:48:14,793 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731026894767 2024-11-08T00:48:14,793 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-08T00:48:14,847 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37429 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=1cbc8c34c2142cc68f3b38bf0794ae6e, source=c302dddf724e,37181,1731026890422, destination=c302dddf724e,32815,1731026890175, warming up region on c302dddf724e,32815,1731026890175 2024-11-08T00:48:14,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37429 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T00:48:14,850 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37429 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=1cbc8c34c2142cc68f3b38bf0794ae6e, source=c302dddf724e,37181,1731026890422, destination=c302dddf724e,32815,1731026890175, running balancer 2024-11-08T00:48:14,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37429 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731026892989, region=1cbc8c34c2142cc68f3b38bf0794ae6e, REOPEN/MOVE 2024-11-08T00:48:14,852 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731026892989, region=1cbc8c34c2142cc68f3b38bf0794ae6e, REOPEN/MOVE 2024-11-08T00:48:14,853 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55175, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T00:48:14,854 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32815 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:14,854 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1cbc8c34c2142cc68f3b38bf0794ae6e, regionState=CLOSING, regionLocation=c302dddf724e,37181,1731026890422 2024-11-08T00:48:14,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32815 {}] regionserver.HRegion(7855): Warmup {ENCODED => 1cbc8c34c2142cc68f3b38bf0794ae6e, NAME => 'testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e.', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:48:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32815 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:48:14,856 INFO [StoreOpener-1cbc8c34c2142cc68f3b38bf0794ae6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:14,857 INFO [StoreOpener-1cbc8c34c2142cc68f3b38bf0794ae6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1cbc8c34c2142cc68f3b38bf0794ae6e columnFamilyName family 2024-11-08T00:48:14,857 DEBUG [StoreOpener-1cbc8c34c2142cc68f3b38bf0794ae6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:14,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731026892989, region=1cbc8c34c2142cc68f3b38bf0794ae6e, REOPEN/MOVE because future has completed 2024-11-08T00:48:14,858 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-08T00:48:14,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1cbc8c34c2142cc68f3b38bf0794ae6e, server=c302dddf724e,37181,1731026890422}] 2024-11-08T00:48:14,859 DEBUG [hfile-prefetch-1731026893821 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e/family/2c449b08bd9d43c19af37b0bdec7ee3f, with total blocks 2, and data blocks 1 2024-11-08T00:48:14,859 DEBUG [hfile-prefetch-1731026893821 {}] bucket.BucketCache(2286): iterating over 1 entries in the backing map 2024-11-08T00:48:14,859 DEBUG [hfile-prefetch-1731026893821 {}] bucket.BucketCache(2294): found block for file 2c449b08bd9d43c19af37b0bdec7ee3f in the backing map. Acquiring read lock for offset 0 2024-11-08T00:48:14,859 DEBUG [hfile-prefetch-1731026893821 {}] bucket.BucketCache(2306): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e/family/2c449b08bd9d43c19af37b0bdec7ee3f has now been fully cached. 2024-11-08T00:48:14,860 DEBUG [hfile-prefetch-1731026893821 {}] hfile.PrefetchExecutor(121): Prefetch completed for 2c449b08bd9d43c19af37b0bdec7ee3f 2024-11-08T00:48:14,860 DEBUG [StoreOpener-1cbc8c34c2142cc68f3b38bf0794ae6e-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e/family/2c449b08bd9d43c19af37b0bdec7ee3f 2024-11-08T00:48:14,860 INFO [StoreOpener-1cbc8c34c2142cc68f3b38bf0794ae6e-1 {}] regionserver.HStore(327): Store=1cbc8c34c2142cc68f3b38bf0794ae6e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:48:14,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32815 {}] regionserver.HRegion(1722): Closing 1cbc8c34c2142cc68f3b38bf0794ae6e, disabling compactions & flushes 2024-11-08T00:48:14,861 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32815 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:14,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32815 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:14,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32815 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. after waiting 0 ms 2024-11-08T00:48:14,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32815 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:14,862 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32815 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32815 {}] regionserver.HRegion(1676): Region close journal for 1cbc8c34c2142cc68f3b38bf0794ae6e: Waiting for close lock at 1731026894860Disabling compacts and flushes for region at 1731026894860Disabling writes for close at 1731026894861 (+1 ms)Writing region close event to WAL at 1731026894862 (+1 ms)Closed at 1731026894862 2024-11-08T00:48:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37429 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-08T00:48:15,016 INFO [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:15,016 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-08T00:48:15,017 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 1cbc8c34c2142cc68f3b38bf0794ae6e, disabling compactions & flushes 2024-11-08T00:48:15,017 INFO [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:15,017 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:15,017 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. after waiting 0 ms 2024-11-08T00:48:15,017 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:15,019 DEBUG [StoreFileCloser-1cbc8c34c2142cc68f3b38bf0794ae6e-family-1 {}] bucket.BucketCache(769): Removing file 2c449b08bd9d43c19af37b0bdec7ee3f for region 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:15,020 DEBUG [StoreFileCloser-1cbc8c34c2142cc68f3b38bf0794ae6e-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file 2c449b08bd9d43c19af37b0bdec7ee3f, starting offset: 0, end offset: 9223372036854775807 2024-11-08T00:48:15,021 DEBUG [StoreFileCloser-1cbc8c34c2142cc68f3b38bf0794ae6e-family-1 {}] bucket.BucketCache(857): removed key 2c449b08bd9d43c19af37b0bdec7ee3f_0 from back map with offset lock 2094080 in the evict process 2024-11-08T00:48:15,027 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-08T00:48:15,028 INFO [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:15,029 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 1cbc8c34c2142cc68f3b38bf0794ae6e: Waiting for close lock at 1731026895017Running coprocessor pre-close hooks at 1731026895017Disabling compacts and flushes for region at 1731026895017Disabling writes for close at 1731026895017Writing region close event to WAL at 1731026895021 (+4 ms)Running coprocessor post-close hooks at 1731026895027 (+6 ms)Closed at 1731026895028 (+1 ms) 2024-11-08T00:48:15,029 INFO [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 1cbc8c34c2142cc68f3b38bf0794ae6e move to c302dddf724e,32815,1731026890175 record at close sequenceid=6 2024-11-08T00:48:15,039 INFO [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:15,039 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1cbc8c34c2142cc68f3b38bf0794ae6e, regionState=CLOSED 2024-11-08T00:48:15,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1cbc8c34c2142cc68f3b38bf0794ae6e, server=c302dddf724e,37181,1731026890422 because future has completed 2024-11-08T00:48:15,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-08T00:48:15,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 1cbc8c34c2142cc68f3b38bf0794ae6e, server=c302dddf724e,37181,1731026890422 in 186 msec 2024-11-08T00:48:15,048 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731026892989, region=1cbc8c34c2142cc68f3b38bf0794ae6e, REOPEN/MOVE; state=CLOSED, location=c302dddf724e,32815,1731026890175; forceNewPlan=false, retain=false 2024-11-08T00:48:15,094 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-08T00:48:15,094 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-08T00:48:15,094 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T00:48:15,094 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T00:48:15,096 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-08T00:48:15,096 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-08T00:48:15,096 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T00:48:15,096 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T00:48:15,099 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T00:48:15,099 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731026895094 2024-11-08T00:48:15,099 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-08T00:48:15,100 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T00:48:15,100 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731026895096 2024-11-08T00:48:15,100 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-08T00:48:15,198 INFO [c302dddf724e:37429 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-08T00:48:15,199 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1cbc8c34c2142cc68f3b38bf0794ae6e, regionState=OPENING, regionLocation=c302dddf724e,32815,1731026890175 2024-11-08T00:48:15,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731026892989, region=1cbc8c34c2142cc68f3b38bf0794ae6e, REOPEN/MOVE because future has completed 2024-11-08T00:48:15,204 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cbc8c34c2142cc68f3b38bf0794ae6e, server=c302dddf724e,32815,1731026890175}] 2024-11-08T00:48:15,360 INFO [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:15,361 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1cbc8c34c2142cc68f3b38bf0794ae6e, NAME => 'testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e.', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:48:15,361 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731026892989 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:15,361 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:48:15,362 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:15,362 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:15,363 INFO [StoreOpener-1cbc8c34c2142cc68f3b38bf0794ae6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:15,364 INFO [StoreOpener-1cbc8c34c2142cc68f3b38bf0794ae6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1cbc8c34c2142cc68f3b38bf0794ae6e columnFamilyName family 2024-11-08T00:48:15,364 DEBUG [StoreOpener-1cbc8c34c2142cc68f3b38bf0794ae6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:15,366 DEBUG [StoreFileOpener-1cbc8c34c2142cc68f3b38bf0794ae6e-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e/family/2c449b08bd9d43c19af37b0bdec7ee3f, delay=912 ms 2024-11-08T00:48:15,367 DEBUG [StoreOpener-1cbc8c34c2142cc68f3b38bf0794ae6e-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e/family/2c449b08bd9d43c19af37b0bdec7ee3f 2024-11-08T00:48:15,368 INFO [StoreOpener-1cbc8c34c2142cc68f3b38bf0794ae6e-1 {}] regionserver.HStore(327): Store=1cbc8c34c2142cc68f3b38bf0794ae6e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:48:15,368 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:15,369 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:15,369 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:15,369 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:15,369 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:15,373 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:15,374 INFO [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 1cbc8c34c2142cc68f3b38bf0794ae6e; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69460979, jitterRate=0.03504924476146698}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:48:15,374 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:15,375 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 1cbc8c34c2142cc68f3b38bf0794ae6e: Running coprocessor pre-open hook at 1731026895362Writing region info on filesystem at 1731026895362Initializing all the Stores at 1731026895362Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026895363 (+1 ms)Cleaning up temporary data from old regions at 1731026895369 (+6 ms)Running coprocessor post-open hooks at 1731026895374 (+5 ms)Region opened successfully at 1731026895375 (+1 ms) 2024-11-08T00:48:15,376 INFO [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e., pid=9, masterSystemTime=1731026895357 2024-11-08T00:48:15,379 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:15,379 INFO [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:15,379 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1cbc8c34c2142cc68f3b38bf0794ae6e, regionState=OPEN, openSeqNum=10, regionLocation=c302dddf724e,32815,1731026890175 2024-11-08T00:48:15,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cbc8c34c2142cc68f3b38bf0794ae6e, server=c302dddf724e,32815,1731026890175 because future has completed 2024-11-08T00:48:15,385 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-08T00:48:15,385 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 1cbc8c34c2142cc68f3b38bf0794ae6e, server=c302dddf724e,32815,1731026890175 in 179 msec 2024-11-08T00:48:15,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731026892989, region=1cbc8c34c2142cc68f3b38bf0794ae6e, REOPEN/MOVE in 534 msec 2024-11-08T00:48:15,405 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T00:48:15,405 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T00:48:15,405 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:48:15,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:48:15,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:48:15,411 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:48:15,411 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T00:48:15,411 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1135712685, stopped=false 2024-11-08T00:48:15,411 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c302dddf724e,37429,1731026889241 2024-11-08T00:48:15,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T00:48:15,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37181-0x100f50c136e0002, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T00:48:15,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:15,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37181-0x100f50c136e0002, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:15,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T00:48:15,415 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:48:15,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:15,415 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T00:48:15,416 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:48:15,416 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37181-0x100f50c136e0002, quorum=127.0.0.1:64158, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:48:15,416 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:48:15,416 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:48:15,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:48:15,417 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c302dddf724e,32815,1731026890175' ***** 2024-11-08T00:48:15,417 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T00:48:15,417 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c302dddf724e,37181,1731026890422' ***** 2024-11-08T00:48:15,417 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T00:48:15,417 INFO [RS:0;c302dddf724e:32815 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T00:48:15,417 INFO [RS:1;c302dddf724e:37181 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T00:48:15,417 INFO [RS:1;c302dddf724e:37181 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/bucket.persistence 2024-11-08T00:48:15,417 INFO [RS:0;c302dddf724e:32815 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/bucket.persistence 2024-11-08T00:48:15,417 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T00:48:15,417 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T00:48:15,418 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T00:48:15,418 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T00:48:15,418 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T00:48:15,418 INFO [RS:0;c302dddf724e:32815 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T00:48:15,418 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T00:48:15,418 INFO [RS:1;c302dddf724e:37181 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T00:48:15,418 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T00:48:15,418 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T00:48:15,428 DEBUG [RS:0;c302dddf724e:32815 {}] bucket.BucketCache(1488): Thread RS:0;c302dddf724e:32815 started persisting bucket cache to file 2024-11-08T00:48:15,428 DEBUG [RS:0;c302dddf724e:32815 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T00:48:15,428 DEBUG [RS:1;c302dddf724e:37181 {}] bucket.BucketCache(1488): Thread RS:1;c302dddf724e:37181 started persisting bucket cache to file 2024-11-08T00:48:15,428 DEBUG [RS:0;c302dddf724e:32815 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T00:48:15,428 DEBUG [RS:1;c302dddf724e:37181 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T00:48:15,428 DEBUG [RS:1;c302dddf724e:37181 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T00:48:15,432 DEBUG [RS:0;c302dddf724e:32815 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T00:48:15,432 DEBUG [RS:1;c302dddf724e:37181 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T00:48:15,432 DEBUG [RS:0;c302dddf724e:32815 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731026895428 2024-11-08T00:48:15,432 DEBUG [RS:1;c302dddf724e:37181 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731026895428 2024-11-08T00:48:15,432 DEBUG [RS:0;c302dddf724e:32815 {}] bucket.BucketCache(1510): Thread RS:0;c302dddf724e:32815 finished persisting bucket cache to file, renaming 2024-11-08T00:48:15,432 DEBUG [RS:1;c302dddf724e:37181 {}] bucket.BucketCache(1510): Thread RS:1;c302dddf724e:37181 finished persisting bucket cache to file, renaming 2024-11-08T00:48:15,432 WARN [RS:1;c302dddf724e:37181 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-08T00:48:15,432 INFO [RS:0;c302dddf724e:32815 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T00:48:15,432 INFO [RS:1;c302dddf724e:37181 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T00:48:15,432 INFO [RS:0;c302dddf724e:32815 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T00:48:15,432 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T00:48:15,432 INFO [RS:1;c302dddf724e:37181 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T00:48:15,432 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T00:48:15,433 INFO [RS:1;c302dddf724e:37181 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T00:48:15,433 INFO [RS:0;c302dddf724e:32815 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T00:48:15,433 INFO [RS:1;c302dddf724e:37181 {}] regionserver.HRegionServer(959): stopping server c302dddf724e,37181,1731026890422 2024-11-08T00:48:15,433 INFO [RS:0;c302dddf724e:32815 {}] regionserver.HRegionServer(3091): Received CLOSE for 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:15,433 INFO [RS:1;c302dddf724e:37181 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T00:48:15,433 INFO [RS:1;c302dddf724e:37181 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;c302dddf724e:37181. 2024-11-08T00:48:15,433 DEBUG [RS:1;c302dddf724e:37181 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:48:15,433 DEBUG [RS:1;c302dddf724e:37181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:48:15,433 INFO [RS:1;c302dddf724e:37181 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T00:48:15,433 INFO [RS:1;c302dddf724e:37181 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T00:48:15,433 INFO [RS:1;c302dddf724e:37181 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T00:48:15,434 INFO [RS:1;c302dddf724e:37181 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T00:48:15,434 INFO [RS:0;c302dddf724e:32815 {}] regionserver.HRegionServer(959): stopping server c302dddf724e,32815,1731026890175 2024-11-08T00:48:15,434 INFO [RS:0;c302dddf724e:32815 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T00:48:15,434 INFO [RS:0;c302dddf724e:32815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c302dddf724e:32815. 2024-11-08T00:48:15,434 DEBUG [RS:0;c302dddf724e:32815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:48:15,434 DEBUG [RS:0;c302dddf724e:32815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:48:15,434 INFO [RS:1;c302dddf724e:37181 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T00:48:15,434 DEBUG [RS:1;c302dddf724e:37181 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-08T00:48:15,434 INFO [RS:0;c302dddf724e:32815 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T00:48:15,434 DEBUG [RS:0;c302dddf724e:32815 {}] regionserver.HRegionServer(1325): Online Regions={1cbc8c34c2142cc68f3b38bf0794ae6e=testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e.} 2024-11-08T00:48:15,434 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T00:48:15,434 INFO [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T00:48:15,434 DEBUG [RS:1;c302dddf724e:37181 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T00:48:15,434 DEBUG [RS:0;c302dddf724e:32815 {}] regionserver.HRegionServer(1351): Waiting on 1cbc8c34c2142cc68f3b38bf0794ae6e 2024-11-08T00:48:15,434 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1cbc8c34c2142cc68f3b38bf0794ae6e, disabling compactions & flushes 2024-11-08T00:48:15,434 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T00:48:15,435 INFO [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:15,435 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T00:48:15,435 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:15,435 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T00:48:15,435 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. after waiting 0 ms 2024-11-08T00:48:15,435 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T00:48:15,435 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:15,435 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T00:48:15,435 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-08T00:48:15,435 DEBUG [StoreFileCloser-1cbc8c34c2142cc68f3b38bf0794ae6e-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e/family/2c449b08bd9d43c19af37b0bdec7ee3f 2024-11-08T00:48:15,440 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnRegionMove1731026892989/1cbc8c34c2142cc68f3b38bf0794ae6e/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-08T00:48:15,441 INFO [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:15,441 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1cbc8c34c2142cc68f3b38bf0794ae6e: Waiting for close lock at 1731026895434Running coprocessor pre-close hooks at 1731026895434Disabling compacts and flushes for region at 1731026895434Disabling writes for close at 1731026895435 (+1 ms)Writing region close event to WAL at 1731026895436 (+1 ms)Running coprocessor post-close hooks at 1731026895441 (+5 ms)Closed at 1731026895441 2024-11-08T00:48:15,441 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e. 2024-11-08T00:48:15,446 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 45c938bbb5bf47e2bfb3c5b3ba52309c 2024-11-08T00:48:15,464 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/.tmp/info/45c938bbb5bf47e2bfb3c5b3ba52309c is 205, key is testBlockEvictionOnRegionMove1731026892989,,1731026893008.1cbc8c34c2142cc68f3b38bf0794ae6e./info:regioninfo/1731026895379/Put/seqid=0 2024-11-08T00:48:15,465 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/.tmp/info/45c938bbb5bf47e2bfb3c5b3ba52309c 2024-11-08T00:48:15,474 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6b5949a57c9947ebac9665fae5cb38e2 2024-11-08T00:48:15,488 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/.tmp/ns/6b5949a57c9947ebac9665fae5cb38e2 is 43, key is default/ns:d/1731026892784/Put/seqid=0 2024-11-08T00:48:15,489 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/.tmp/ns/6b5949a57c9947ebac9665fae5cb38e2 2024-11-08T00:48:15,497 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 75ec059487f54b2c9024d4a20779d699 2024-11-08T00:48:15,511 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/.tmp/table/75ec059487f54b2c9024d4a20779d699 is 78, key is testBlockEvictionOnRegionMove1731026892989/table:state/1731026893447/Put/seqid=0 2024-11-08T00:48:15,512 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/.tmp/table/75ec059487f54b2c9024d4a20779d699 2024-11-08T00:48:15,515 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/.tmp/info/45c938bbb5bf47e2bfb3c5b3ba52309c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/info/45c938bbb5bf47e2bfb3c5b3ba52309c 2024-11-08T00:48:15,516 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/info/45c938bbb5bf47e2bfb3c5b3ba52309c, delay=940 ms 2024-11-08T00:48:15,517 INFO [regionserver/c302dddf724e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T00:48:15,517 INFO [regionserver/c302dddf724e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T00:48:15,518 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/info/45c938bbb5bf47e2bfb3c5b3ba52309c, entries=15, sequenceid=15, filesize=7.6 K 2024-11-08T00:48:15,518 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/.tmp/ns/6b5949a57c9947ebac9665fae5cb38e2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/ns/6b5949a57c9947ebac9665fae5cb38e2 2024-11-08T00:48:15,519 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/ns/6b5949a57c9947ebac9665fae5cb38e2, delay=910 ms 2024-11-08T00:48:15,521 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/ns/6b5949a57c9947ebac9665fae5cb38e2, entries=2, sequenceid=15, filesize=5.0 K 2024-11-08T00:48:15,521 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/.tmp/table/75ec059487f54b2c9024d4a20779d699 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/table/75ec059487f54b2c9024d4a20779d699 2024-11-08T00:48:15,522 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/table/75ec059487f54b2c9024d4a20779d699, delay=978 ms 2024-11-08T00:48:15,523 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/table/75ec059487f54b2c9024d4a20779d699, entries=2, sequenceid=15, filesize=5.3 K 2024-11-08T00:48:15,524 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 89ms, sequenceid=15, compaction requested=false 2024-11-08T00:48:15,524 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T00:48:15,526 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/info/45c938bbb5bf47e2bfb3c5b3ba52309c 2024-11-08T00:48:15,526 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/ns/6b5949a57c9947ebac9665fae5cb38e2 2024-11-08T00:48:15,527 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/table/75ec059487f54b2c9024d4a20779d699 2024-11-08T00:48:15,531 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-08T00:48:15,532 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T00:48:15,532 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T00:48:15,533 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731026895434Running coprocessor pre-close hooks at 1731026895434Disabling compacts and flushes for region at 1731026895434Disabling writes for close at 1731026895435 (+1 ms)Obtaining lock to block concurrent updates at 1731026895435Preparing flush snapshotting stores in 1588230740 at 1731026895435Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731026895436 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731026895436Flushing 1588230740/info: creating writer at 1731026895436Flushing 1588230740/info: appending metadata at 1731026895463 (+27 ms)Flushing 1588230740/info: closing flushed file at 1731026895463Flushing 1588230740/ns: creating writer at 1731026895468 (+5 ms)Flushing 1588230740/ns: appending metadata at 1731026895487 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731026895487Flushing 1588230740/table: creating writer at 1731026895491 (+4 ms)Flushing 1588230740/table: appending metadata at 1731026895510 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731026895511 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@729ad84d: reopening flushed file at 1731026895515 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b42e6c3: reopening flushed file at 1731026895518 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bdb0839: reopening flushed file at 1731026895521 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 89ms, sequenceid=15, compaction requested=false at 1731026895524 (+3 ms)Writing region close event to WAL at 1731026895527 (+3 ms)Running coprocessor post-close hooks at 1731026895532 (+5 ms)Closed at 1731026895532 2024-11-08T00:48:15,533 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T00:48:15,635 INFO [RS:1;c302dddf724e:37181 {}] regionserver.HRegionServer(976): stopping server c302dddf724e,37181,1731026890422; all regions closed. 2024-11-08T00:48:15,635 INFO [RS:0;c302dddf724e:32815 {}] regionserver.HRegionServer(976): stopping server c302dddf724e,32815,1731026890175; all regions closed. 2024-11-08T00:48:15,640 DEBUG [RS:0;c302dddf724e:32815 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/oldWALs 2024-11-08T00:48:15,640 DEBUG [RS:1;c302dddf724e:37181 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/oldWALs 2024-11-08T00:48:15,641 INFO [RS:1;c302dddf724e:37181 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c302dddf724e%2C37181%2C1731026890422.meta:.meta(num 1731026892629) 2024-11-08T00:48:15,641 INFO [RS:0;c302dddf724e:32815 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c302dddf724e%2C32815%2C1731026890175:(num 1731026892194) 2024-11-08T00:48:15,641 DEBUG [RS:0;c302dddf724e:32815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:48:15,641 INFO [RS:0;c302dddf724e:32815 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T00:48:15,641 INFO [RS:0;c302dddf724e:32815 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T00:48:15,641 INFO [RS:0;c302dddf724e:32815 {}] hbase.ChoreService(370): Chore service for: regionserver/c302dddf724e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T00:48:15,641 INFO [RS:0;c302dddf724e:32815 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T00:48:15,641 INFO [RS:0;c302dddf724e:32815 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T00:48:15,641 INFO [regionserver/c302dddf724e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T00:48:15,642 INFO [RS:0;c302dddf724e:32815 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T00:48:15,642 INFO [RS:0;c302dddf724e:32815 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T00:48:15,642 INFO [RS:0;c302dddf724e:32815 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32815 2024-11-08T00:48:15,643 DEBUG [RS:1;c302dddf724e:37181 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/oldWALs 2024-11-08T00:48:15,643 INFO [RS:1;c302dddf724e:37181 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c302dddf724e%2C37181%2C1731026890422:(num 1731026892196) 2024-11-08T00:48:15,643 DEBUG [RS:1;c302dddf724e:37181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:48:15,643 INFO [RS:1;c302dddf724e:37181 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T00:48:15,643 INFO [RS:1;c302dddf724e:37181 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T00:48:15,643 INFO [RS:1;c302dddf724e:37181 {}] hbase.ChoreService(370): Chore service for: regionserver/c302dddf724e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T00:48:15,643 INFO [RS:1;c302dddf724e:37181 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T00:48:15,644 INFO [regionserver/c302dddf724e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T00:48:15,644 INFO [RS:1;c302dddf724e:37181 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37181 2024-11-08T00:48:15,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37181-0x100f50c136e0002, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c302dddf724e,37181,1731026890422 2024-11-08T00:48:15,647 INFO [RS:1;c302dddf724e:37181 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T00:48:15,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T00:48:15,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c302dddf724e,32815,1731026890175 2024-11-08T00:48:15,648 INFO [RS:0;c302dddf724e:32815 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T00:48:15,650 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c302dddf724e,32815,1731026890175] 2024-11-08T00:48:15,652 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c302dddf724e,32815,1731026890175 already deleted, retry=false 2024-11-08T00:48:15,653 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c302dddf724e,32815,1731026890175 expired; onlineServers=1 2024-11-08T00:48:15,653 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c302dddf724e,37181,1731026890422] 2024-11-08T00:48:15,654 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c302dddf724e,37181,1731026890422 already deleted, retry=false 2024-11-08T00:48:15,654 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c302dddf724e,37181,1731026890422 expired; onlineServers=0 2024-11-08T00:48:15,654 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c302dddf724e,37429,1731026889241' ***** 2024-11-08T00:48:15,655 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T00:48:15,655 INFO [M:0;c302dddf724e:37429 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T00:48:15,655 INFO [M:0;c302dddf724e:37429 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T00:48:15,655 DEBUG [M:0;c302dddf724e:37429 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T00:48:15,655 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T00:48:15,655 DEBUG [M:0;c302dddf724e:37429 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T00:48:15,655 DEBUG [master/c302dddf724e:0:becomeActiveMaster-HFileCleaner.small.0-1731026891809 {}] cleaner.HFileCleaner(306): Exit Thread[master/c302dddf724e:0:becomeActiveMaster-HFileCleaner.small.0-1731026891809,5,FailOnTimeoutGroup] 2024-11-08T00:48:15,655 DEBUG [master/c302dddf724e:0:becomeActiveMaster-HFileCleaner.large.0-1731026891808 {}] cleaner.HFileCleaner(306): Exit Thread[master/c302dddf724e:0:becomeActiveMaster-HFileCleaner.large.0-1731026891808,5,FailOnTimeoutGroup] 2024-11-08T00:48:15,655 INFO [M:0;c302dddf724e:37429 {}] hbase.ChoreService(370): Chore service for: master/c302dddf724e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T00:48:15,656 INFO [M:0;c302dddf724e:37429 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T00:48:15,656 DEBUG [M:0;c302dddf724e:37429 {}] master.HMaster(1795): Stopping service threads 2024-11-08T00:48:15,656 INFO [M:0;c302dddf724e:37429 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T00:48:15,656 INFO [M:0;c302dddf724e:37429 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:48:15,657 INFO [M:0;c302dddf724e:37429 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:48:15,657 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T00:48:15,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T00:48:15,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:15,658 DEBUG [M:0;c302dddf724e:37429 {}] zookeeper.ZKUtil(347): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T00:48:15,658 WARN [M:0;c302dddf724e:37429 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T00:48:15,658 INFO [M:0;c302dddf724e:37429 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/.lastflushedseqids 2024-11-08T00:48:15,668 INFO [M:0;c302dddf724e:37429 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:48:15,668 INFO [M:0;c302dddf724e:37429 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T00:48:15,668 DEBUG [M:0;c302dddf724e:37429 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T00:48:15,668 INFO [M:0;c302dddf724e:37429 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:48:15,668 DEBUG [M:0;c302dddf724e:37429 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:48:15,668 DEBUG [M:0;c302dddf724e:37429 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T00:48:15,668 DEBUG [M:0;c302dddf724e:37429 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:48:15,669 INFO [M:0;c302dddf724e:37429 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-08T00:48:15,680 DEBUG [M:0;c302dddf724e:37429 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b13a4992abd74d7bb1da9c137f2afb55 2024-11-08T00:48:15,694 DEBUG [M:0;c302dddf724e:37429 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b13a4992abd74d7bb1da9c137f2afb55 is 82, key is hbase:meta,,1/info:regioninfo/1731026892714/Put/seqid=0 2024-11-08T00:48:15,695 INFO [M:0;c302dddf724e:37429 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b13a4992abd74d7bb1da9c137f2afb55 2024-11-08T00:48:15,703 DEBUG [M:0;c302dddf724e:37429 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8cb14ea7cc2c4140b206b363ace4373e 2024-11-08T00:48:15,719 DEBUG [M:0;c302dddf724e:37429 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8cb14ea7cc2c4140b206b363ace4373e is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731026893454/Put/seqid=0 2024-11-08T00:48:15,720 INFO [M:0;c302dddf724e:37429 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8cb14ea7cc2c4140b206b363ace4373e 2024-11-08T00:48:15,729 DEBUG [M:0;c302dddf724e:37429 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b8db8ec0263c40ecb038b8090e2f43e7 2024-11-08T00:48:15,743 DEBUG [M:0;c302dddf724e:37429 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b8db8ec0263c40ecb038b8090e2f43e7 is 69, key is c302dddf724e,32815,1731026890175/rs:state/1731026891875/Put/seqid=0 2024-11-08T00:48:15,743 INFO [M:0;c302dddf724e:37429 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b8db8ec0263c40ecb038b8090e2f43e7 2024-11-08T00:48:15,745 DEBUG [M:0;c302dddf724e:37429 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b13a4992abd74d7bb1da9c137f2afb55 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b13a4992abd74d7bb1da9c137f2afb55 2024-11-08T00:48:15,748 INFO [M:0;c302dddf724e:37429 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b13a4992abd74d7bb1da9c137f2afb55, entries=8, sequenceid=84, filesize=5.5 K 2024-11-08T00:48:15,748 DEBUG [M:0;c302dddf724e:37429 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8cb14ea7cc2c4140b206b363ace4373e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8cb14ea7cc2c4140b206b363ace4373e 2024-11-08T00:48:15,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37181-0x100f50c136e0002, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:48:15,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37181-0x100f50c136e0002, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:48:15,751 INFO [M:0;c302dddf724e:37429 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8cb14ea7cc2c4140b206b363ace4373e, entries=9, sequenceid=84, filesize=6.5 K 2024-11-08T00:48:15,751 INFO [RS:1;c302dddf724e:37181 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T00:48:15,751 INFO [RS:1;c302dddf724e:37181 {}] regionserver.HRegionServer(1031): Exiting; stopping=c302dddf724e,37181,1731026890422; zookeeper connection closed. 2024-11-08T00:48:15,751 DEBUG [M:0;c302dddf724e:37429 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b8db8ec0263c40ecb038b8090e2f43e7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b8db8ec0263c40ecb038b8090e2f43e7 2024-11-08T00:48:15,751 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@a1c6b80 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@a1c6b80 2024-11-08T00:48:15,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:48:15,752 INFO [RS:0;c302dddf724e:32815 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T00:48:15,752 INFO [RS:0;c302dddf724e:32815 {}] regionserver.HRegionServer(1031): Exiting; stopping=c302dddf724e,32815,1731026890175; zookeeper connection closed. 2024-11-08T00:48:15,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32815-0x100f50c136e0001, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:48:15,752 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@69ad029 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@69ad029 2024-11-08T00:48:15,753 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-08T00:48:15,754 INFO [M:0;c302dddf724e:37429 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b8db8ec0263c40ecb038b8090e2f43e7, entries=2, sequenceid=84, filesize=5.1 K 2024-11-08T00:48:15,755 INFO [M:0;c302dddf724e:37429 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 87ms, sequenceid=84, compaction requested=false 2024-11-08T00:48:15,758 INFO [M:0;c302dddf724e:37429 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:48:15,759 DEBUG [M:0;c302dddf724e:37429 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731026895668Disabling compacts and flushes for region at 1731026895668Disabling writes for close at 1731026895668Obtaining lock to block concurrent updates at 1731026895669 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731026895669Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731026895669Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731026895669Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731026895670 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731026895694 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731026895694Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731026895697 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731026895718 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731026895718Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731026895722 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731026895742 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731026895742Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d6abba8: reopening flushed file at 1731026895745 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76d732a1: reopening flushed file at 1731026895748 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f7fc4c9: reopening flushed file at 1731026895751 (+3 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 87ms, sequenceid=84, compaction requested=false at 1731026895755 (+4 ms)Writing region close event to WAL at 1731026895758 (+3 ms)Closed at 1731026895758 2024-11-08T00:48:15,763 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T00:48:15,763 INFO [M:0;c302dddf724e:37429 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T00:48:15,763 INFO [M:0;c302dddf724e:37429 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37429 2024-11-08T00:48:15,764 INFO [M:0;c302dddf724e:37429 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T00:48:15,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:48:15,867 INFO [M:0;c302dddf724e:37429 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T00:48:15,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x100f50c136e0000, quorum=127.0.0.1:64158, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:48:15,872 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T00:48:15,892 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-08T00:48:15,892 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:15,900 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/c302dddf724e:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/c302dddf724e:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/c302dddf724e:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731026895519 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1731026895516 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/c302dddf724e:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: hfile-prefetch-1731026895366 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731026893821 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=803 (was 829), ProcessCount=11 (was 11), AvailableMemoryMB=3122 (was 922) - AvailableMemoryMB LEAK? - 2024-11-08T00:48:15,906 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=803, ProcessCount=11, AvailableMemoryMB=3122 2024-11-08T00:48:15,906 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/hadoop-log-dir so I do NOT create it in target/test-data/be782f04-237a-9a39-9d79-99c0a8908307 2024-11-08T00:48:15,906 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/hadoop-tmp-dir so I do NOT create it in target/test-data/be782f04-237a-9a39-9d79-99c0a8908307 2024-11-08T00:48:15,906 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:15,909 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be782f04-237a-9a39-9d79-99c0a8908307/cluster_6af363ee-67d7-7308-6bb4-6222dc9deee7, deleteOnExit=true 2024-11-08T00:48:15,912 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be782f04-237a-9a39-9d79-99c0a8908307/cluster_6af363ee-67d7-7308-6bb4-6222dc9deee7/zookeeper_0, clientPort=58229, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be782f04-237a-9a39-9d79-99c0a8908307/cluster_6af363ee-67d7-7308-6bb4-6222dc9deee7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be782f04-237a-9a39-9d79-99c0a8908307/cluster_6af363ee-67d7-7308-6bb4-6222dc9deee7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T00:48:15,913 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58229 2024-11-08T00:48:15,928 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77 with version=8 2024-11-08T00:48:15,928 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/hbase-staging 2024-11-08T00:48:15,931 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c302dddf724e:0 server-side Connection retries=45 2024-11-08T00:48:15,931 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:15,932 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:15,932 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T00:48:15,932 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:15,932 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T00:48:15,932 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T00:48:15,932 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T00:48:15,933 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36775 2024-11-08T00:48:15,934 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36775 connecting to ZooKeeper ensemble=127.0.0.1:58229 2024-11-08T00:48:15,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:367750x0, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T00:48:15,943 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36775-0x100f50c2eb70000 connected 2024-11-08T00:48:15,970 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:15,970 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:15,971 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:48:15,971 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77, hbase.cluster.distributed=false 2024-11-08T00:48:15,973 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T00:48:15,974 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36775 2024-11-08T00:48:15,974 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36775 2024-11-08T00:48:15,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36775 2024-11-08T00:48:15,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36775 2024-11-08T00:48:15,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36775 2024-11-08T00:48:15,993 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c302dddf724e:0 server-side Connection retries=45 2024-11-08T00:48:15,994 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:15,994 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:15,994 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T00:48:15,994 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:15,994 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T00:48:15,994 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T00:48:15,994 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T00:48:15,995 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45629 2024-11-08T00:48:15,996 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45629 connecting to ZooKeeper ensemble=127.0.0.1:58229 2024-11-08T00:48:15,997 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:15,997 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:16,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:456290x0, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T00:48:16,000 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45629-0x100f50c2eb70001 connected 2024-11-08T00:48:16,000 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:48:16,001 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T00:48:16,003 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be782f04-237a-9a39-9d79-99c0a8908307/bucket.cache 2024-11-08T00:48:16,003 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T00:48:16,003 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T00:48:16,003 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T00:48:16,004 INFO [Thread-119 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T00:48:16,004 WARN [Thread-119 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T00:48:16,004 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be782f04-237a-9a39-9d79-99c0a8908307/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be782f04-237a-9a39-9d79-99c0a8908307/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T00:48:16,004 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T00:48:16,004 INFO [Thread-119 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be782f04-237a-9a39-9d79-99c0a8908307/bucket.persistence is complete. 2024-11-08T00:48:16,004 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T00:48:16,005 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T00:48:16,006 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T00:48:16,007 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45629 2024-11-08T00:48:16,007 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45629 2024-11-08T00:48:16,007 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45629 2024-11-08T00:48:16,011 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45629 2024-11-08T00:48:16,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45629 2024-11-08T00:48:16,030 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c302dddf724e:0 server-side Connection retries=45 2024-11-08T00:48:16,030 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:16,030 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:16,030 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T00:48:16,030 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:16,030 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T00:48:16,030 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T00:48:16,030 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T00:48:16,031 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44739 2024-11-08T00:48:16,033 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44739 connecting to ZooKeeper ensemble=127.0.0.1:58229 2024-11-08T00:48:16,033 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:16,034 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:16,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:447390x0, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T00:48:16,038 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44739-0x100f50c2eb70002 connected 2024-11-08T00:48:16,038 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44739-0x100f50c2eb70002, quorum=127.0.0.1:58229, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:48:16,038 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T00:48:16,041 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be782f04-237a-9a39-9d79-99c0a8908307/bucket.cache 2024-11-08T00:48:16,041 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T00:48:16,041 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T00:48:16,041 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T00:48:16,042 INFO [Thread-120 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T00:48:16,042 WARN [Thread-120 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T00:48:16,043 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T00:48:16,043 INFO [Thread-120 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be782f04-237a-9a39-9d79-99c0a8908307/bucket.persistence is complete. 2024-11-08T00:48:16,044 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be782f04-237a-9a39-9d79-99c0a8908307/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be782f04-237a-9a39-9d79-99c0a8908307/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T00:48:16,045 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T00:48:16,046 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44739-0x100f50c2eb70002, quorum=127.0.0.1:58229, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T00:48:16,047 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44739-0x100f50c2eb70002, quorum=127.0.0.1:58229, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T00:48:16,052 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44739 2024-11-08T00:48:16,052 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44739 2024-11-08T00:48:16,052 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44739 2024-11-08T00:48:16,054 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44739 2024-11-08T00:48:16,056 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44739 2024-11-08T00:48:16,076 DEBUG [M:0;c302dddf724e:36775 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c302dddf724e:36775 2024-11-08T00:48:16,076 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c302dddf724e,36775,1731026895930 2024-11-08T00:48:16,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44739-0x100f50c2eb70002, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:48:16,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:48:16,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:48:16,080 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c302dddf724e,36775,1731026895930 2024-11-08T00:48:16,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44739-0x100f50c2eb70002, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T00:48:16,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44739-0x100f50c2eb70002, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:16,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T00:48:16,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:16,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:16,084 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T00:48:16,086 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c302dddf724e,36775,1731026895930 from backup master directory 2024-11-08T00:48:16,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c302dddf724e,36775,1731026895930 2024-11-08T00:48:16,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44739-0x100f50c2eb70002, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:48:16,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:48:16,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:48:16,088 WARN [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T00:48:16,089 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c302dddf724e,36775,1731026895930 2024-11-08T00:48:16,090 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/hbase.id] with ID: 339b3297-2b2f-4d2d-b7ca-a35a4fada4b1 2024-11-08T00:48:16,090 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/.tmp/hbase.id 2024-11-08T00:48:16,094 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/hbase.id] 2024-11-08T00:48:16,137 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:16,137 INFO [master/c302dddf724e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T00:48:16,140 INFO [master/c302dddf724e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-08T00:48:16,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:16,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:16,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44739-0x100f50c2eb70002, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:16,149 INFO [master/c302dddf724e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T00:48:16,150 INFO [master/c302dddf724e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T00:48:16,151 INFO [master/c302dddf724e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T00:48:16,178 INFO [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store 2024-11-08T00:48:16,185 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:48:16,185 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T00:48:16,185 INFO [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:48:16,185 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:48:16,185 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T00:48:16,185 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:48:16,186 INFO [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:48:16,186 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731026896185Disabling compacts and flushes for region at 1731026896185Disabling writes for close at 1731026896185Writing region close event to WAL at 1731026896185Closed at 1731026896185 2024-11-08T00:48:16,186 WARN [master/c302dddf724e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/.initializing 2024-11-08T00:48:16,186 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/WALs/c302dddf724e,36775,1731026895930 2024-11-08T00:48:16,193 INFO [master/c302dddf724e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T00:48:16,196 INFO [master/c302dddf724e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c302dddf724e%2C36775%2C1731026895930, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/WALs/c302dddf724e,36775,1731026895930, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/oldWALs, maxLogs=10 2024-11-08T00:48:16,218 INFO [master/c302dddf724e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/WALs/c302dddf724e,36775,1731026895930/c302dddf724e%2C36775%2C1731026895930.1731026896197 2024-11-08T00:48:16,218 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T00:48:16,218 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:48:16,218 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:48:16,218 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:16,219 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:16,219 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:16,226 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T00:48:16,226 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:16,226 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:48:16,227 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:16,231 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T00:48:16,232 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:16,232 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:48:16,232 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:16,237 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T00:48:16,237 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:16,237 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:48:16,238 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:16,242 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T00:48:16,243 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:16,243 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:48:16,243 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:16,244 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:16,244 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:16,244 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:16,245 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:16,245 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T00:48:16,245 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:48:16,253 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:48:16,253 INFO [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73190662, jitterRate=0.09062585234642029}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T00:48:16,255 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731026896219Initializing all the Stores at 1731026896219Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026896219Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026896219Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026896219Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026896219Cleaning up temporary data from old regions at 1731026896245 (+26 ms)Region opened successfully at 1731026896254 (+9 ms) 2024-11-08T00:48:16,255 INFO [master/c302dddf724e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T00:48:16,260 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72cd206e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c302dddf724e/172.17.0.2:0 2024-11-08T00:48:16,261 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T00:48:16,261 INFO [master/c302dddf724e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T00:48:16,261 INFO [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T00:48:16,261 INFO [master/c302dddf724e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:48:16,262 INFO [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:48:16,262 INFO [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-08T00:48:16,262 INFO [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T00:48:16,265 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T00:48:16,266 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T00:48:16,267 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T00:48:16,268 INFO [master/c302dddf724e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T00:48:16,269 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T00:48:16,271 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T00:48:16,271 INFO [master/c302dddf724e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T00:48:16,272 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T00:48:16,273 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T00:48:16,274 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T00:48:16,276 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T00:48:16,279 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T00:48:16,280 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T00:48:16,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T00:48:16,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44739-0x100f50c2eb70002, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T00:48:16,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:16,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T00:48:16,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44739-0x100f50c2eb70002, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:16,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:16,284 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c302dddf724e,36775,1731026895930, sessionid=0x100f50c2eb70000, setting cluster-up flag (Was=false) 2024-11-08T00:48:16,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:16,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:16,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44739-0x100f50c2eb70002, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:16,296 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T00:48:16,297 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c302dddf724e,36775,1731026895930 2024-11-08T00:48:16,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:16,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44739-0x100f50c2eb70002, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:16,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:16,308 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T00:48:16,309 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c302dddf724e,36775,1731026895930 2024-11-08T00:48:16,310 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T00:48:16,312 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:48:16,312 INFO [master/c302dddf724e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T00:48:16,312 INFO [master/c302dddf724e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T00:48:16,312 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c302dddf724e,36775,1731026895930 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T00:48:16,314 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c302dddf724e:0, corePoolSize=5, maxPoolSize=5 2024-11-08T00:48:16,314 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c302dddf724e:0, corePoolSize=5, maxPoolSize=5 2024-11-08T00:48:16,314 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c302dddf724e:0, corePoolSize=5, maxPoolSize=5 2024-11-08T00:48:16,315 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c302dddf724e:0, corePoolSize=5, maxPoolSize=5 2024-11-08T00:48:16,315 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c302dddf724e:0, corePoolSize=10, maxPoolSize=10 2024-11-08T00:48:16,315 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,315 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c302dddf724e:0, corePoolSize=2, maxPoolSize=2 2024-11-08T00:48:16,315 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,317 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T00:48:16,317 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T00:48:16,318 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:16,318 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T00:48:16,319 INFO [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026926319 2024-11-08T00:48:16,319 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T00:48:16,319 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T00:48:16,319 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T00:48:16,319 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T00:48:16,319 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T00:48:16,319 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T00:48:16,324 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,324 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T00:48:16,324 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T00:48:16,324 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T00:48:16,326 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T00:48:16,326 INFO [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T00:48:16,326 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c302dddf724e:0:becomeActiveMaster-HFileCleaner.large.0-1731026896326,5,FailOnTimeoutGroup] 2024-11-08T00:48:16,327 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c302dddf724e:0:becomeActiveMaster-HFileCleaner.small.0-1731026896326,5,FailOnTimeoutGroup] 2024-11-08T00:48:16,327 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,327 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T00:48:16,327 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,327 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,338 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T00:48:16,339 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77 2024-11-08T00:48:16,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:48:16,348 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T00:48:16,352 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T00:48:16,352 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:16,352 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:48:16,353 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T00:48:16,356 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T00:48:16,356 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:16,357 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:48:16,357 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T00:48:16,360 INFO [RS:0;c302dddf724e:45629 {}] regionserver.HRegionServer(746): ClusterId : 339b3297-2b2f-4d2d-b7ca-a35a4fada4b1 2024-11-08T00:48:16,360 DEBUG [RS:0;c302dddf724e:45629 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T00:48:16,360 INFO [RS:1;c302dddf724e:44739 {}] regionserver.HRegionServer(746): ClusterId : 339b3297-2b2f-4d2d-b7ca-a35a4fada4b1 2024-11-08T00:48:16,360 DEBUG [RS:1;c302dddf724e:44739 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T00:48:16,361 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T00:48:16,361 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:16,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:48:16,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T00:48:16,364 DEBUG [RS:0;c302dddf724e:45629 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T00:48:16,364 DEBUG [RS:0;c302dddf724e:45629 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T00:48:16,367 DEBUG [RS:0;c302dddf724e:45629 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T00:48:16,368 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T00:48:16,368 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:16,368 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:48:16,368 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T00:48:16,368 DEBUG [RS:0;c302dddf724e:45629 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b446d6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c302dddf724e/172.17.0.2:0 2024-11-08T00:48:16,369 DEBUG [RS:1;c302dddf724e:44739 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T00:48:16,369 DEBUG [RS:1;c302dddf724e:44739 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T00:48:16,369 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740 2024-11-08T00:48:16,369 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740 2024-11-08T00:48:16,369 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T00:48:16,369 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T00:48:16,370 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T00:48:16,370 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T00:48:16,371 DEBUG [RS:1;c302dddf724e:44739 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T00:48:16,372 DEBUG [RS:1;c302dddf724e:44739 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e232d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c302dddf724e/172.17.0.2:0 2024-11-08T00:48:16,385 DEBUG [RS:1;c302dddf724e:44739 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;c302dddf724e:44739 2024-11-08T00:48:16,385 INFO [RS:1;c302dddf724e:44739 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T00:48:16,385 INFO [RS:1;c302dddf724e:44739 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T00:48:16,385 DEBUG [RS:1;c302dddf724e:44739 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T00:48:16,386 INFO [RS:1;c302dddf724e:44739 {}] regionserver.HRegionServer(2659): reportForDuty to master=c302dddf724e,36775,1731026895930 with port=44739, startcode=1731026896029 2024-11-08T00:48:16,387 DEBUG [RS:1;c302dddf724e:44739 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T00:48:16,389 DEBUG [RS:0;c302dddf724e:45629 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c302dddf724e:45629 2024-11-08T00:48:16,390 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34951, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T00:48:16,390 INFO [RS:0;c302dddf724e:45629 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T00:48:16,390 INFO [RS:0;c302dddf724e:45629 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T00:48:16,390 DEBUG [RS:0;c302dddf724e:45629 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T00:48:16,390 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36775 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c302dddf724e,44739,1731026896029 2024-11-08T00:48:16,390 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36775 {}] master.ServerManager(517): Registering regionserver=c302dddf724e,44739,1731026896029 2024-11-08T00:48:16,391 INFO [RS:0;c302dddf724e:45629 {}] regionserver.HRegionServer(2659): reportForDuty to master=c302dddf724e,36775,1731026895930 with port=45629, startcode=1731026895993 2024-11-08T00:48:16,391 DEBUG [RS:0;c302dddf724e:45629 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T00:48:16,392 DEBUG [RS:1;c302dddf724e:44739 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77 2024-11-08T00:48:16,393 DEBUG [RS:1;c302dddf724e:44739 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T00:48:16,393 DEBUG [RS:1;c302dddf724e:44739 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T00:48:16,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T00:48:16,396 DEBUG [RS:1;c302dddf724e:44739 {}] zookeeper.ZKUtil(111): regionserver:44739-0x100f50c2eb70002, quorum=127.0.0.1:58229, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c302dddf724e,44739,1731026896029 2024-11-08T00:48:16,396 WARN [RS:1;c302dddf724e:44739 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T00:48:16,396 INFO [RS:1;c302dddf724e:44739 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T00:48:16,396 DEBUG [RS:1;c302dddf724e:44739 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,44739,1731026896029 2024-11-08T00:48:16,397 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:48:16,397 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68944164, jitterRate=0.027348101139068604}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T00:48:16,398 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731026896348Initializing all the Stores at 1731026896348Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026896348Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026896348Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026896348Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026896348Cleaning up temporary data from old regions at 1731026896370 (+22 ms)Region opened successfully at 1731026896398 (+28 ms) 2024-11-08T00:48:16,398 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T00:48:16,398 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T00:48:16,398 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T00:48:16,398 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T00:48:16,398 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T00:48:16,399 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c302dddf724e,44739,1731026896029] 2024-11-08T00:48:16,401 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T00:48:16,401 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731026896398Disabling compacts and flushes for region at 1731026896398Disabling writes for close at 1731026896398Writing region close event to WAL at 1731026896400 (+2 ms)Closed at 1731026896400 2024-11-08T00:48:16,401 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52985, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T00:48:16,402 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36775 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c302dddf724e,45629,1731026895993 2024-11-08T00:48:16,402 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T00:48:16,402 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T00:48:16,402 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36775 {}] master.ServerManager(517): Registering regionserver=c302dddf724e,45629,1731026895993 2024-11-08T00:48:16,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T00:48:16,403 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:48:16,404 DEBUG [RS:0;c302dddf724e:45629 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77 2024-11-08T00:48:16,404 DEBUG [RS:0;c302dddf724e:45629 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T00:48:16,404 DEBUG [RS:0;c302dddf724e:45629 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T00:48:16,404 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T00:48:16,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T00:48:16,413 DEBUG [RS:0;c302dddf724e:45629 {}] zookeeper.ZKUtil(111): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c302dddf724e,45629,1731026895993 2024-11-08T00:48:16,413 WARN [RS:0;c302dddf724e:45629 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T00:48:16,413 INFO [RS:0;c302dddf724e:45629 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T00:48:16,414 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c302dddf724e,45629,1731026895993] 2024-11-08T00:48:16,414 DEBUG [RS:0;c302dddf724e:45629 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,45629,1731026895993 2024-11-08T00:48:16,414 INFO [RS:1;c302dddf724e:44739 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T00:48:16,418 INFO [RS:1;c302dddf724e:44739 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T00:48:16,419 INFO [RS:1;c302dddf724e:44739 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T00:48:16,419 INFO [RS:1;c302dddf724e:44739 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,419 INFO [RS:1;c302dddf724e:44739 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T00:48:16,420 INFO [RS:1;c302dddf724e:44739 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T00:48:16,420 INFO [RS:1;c302dddf724e:44739 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,421 DEBUG [RS:1;c302dddf724e:44739 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,421 DEBUG [RS:1;c302dddf724e:44739 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,421 DEBUG [RS:1;c302dddf724e:44739 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,421 DEBUG [RS:1;c302dddf724e:44739 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,421 DEBUG [RS:1;c302dddf724e:44739 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,421 DEBUG [RS:1;c302dddf724e:44739 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c302dddf724e:0, corePoolSize=2, maxPoolSize=2 2024-11-08T00:48:16,421 DEBUG [RS:1;c302dddf724e:44739 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,421 INFO [RS:0;c302dddf724e:45629 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T00:48:16,422 DEBUG [RS:1;c302dddf724e:44739 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,422 DEBUG [RS:1;c302dddf724e:44739 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,422 DEBUG [RS:1;c302dddf724e:44739 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,422 DEBUG [RS:1;c302dddf724e:44739 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,422 DEBUG [RS:1;c302dddf724e:44739 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,422 DEBUG [RS:1;c302dddf724e:44739 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c302dddf724e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T00:48:16,422 DEBUG [RS:1;c302dddf724e:44739 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c302dddf724e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T00:48:16,424 INFO [RS:0;c302dddf724e:45629 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T00:48:16,425 INFO [RS:1;c302dddf724e:44739 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,425 INFO [RS:1;c302dddf724e:44739 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,425 INFO [RS:1;c302dddf724e:44739 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,425 INFO [RS:1;c302dddf724e:44739 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,425 INFO [RS:1;c302dddf724e:44739 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,425 INFO [RS:1;c302dddf724e:44739 {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,44739,1731026896029-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T00:48:16,429 INFO [RS:0;c302dddf724e:45629 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T00:48:16,429 INFO [RS:0;c302dddf724e:45629 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,430 INFO [RS:0;c302dddf724e:45629 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T00:48:16,431 INFO [RS:0;c302dddf724e:45629 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T00:48:16,431 INFO [RS:0;c302dddf724e:45629 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,431 DEBUG [RS:0;c302dddf724e:45629 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,431 DEBUG [RS:0;c302dddf724e:45629 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,431 DEBUG [RS:0;c302dddf724e:45629 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,431 DEBUG [RS:0;c302dddf724e:45629 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,432 DEBUG [RS:0;c302dddf724e:45629 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,432 DEBUG [RS:0;c302dddf724e:45629 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c302dddf724e:0, corePoolSize=2, maxPoolSize=2 2024-11-08T00:48:16,432 DEBUG [RS:0;c302dddf724e:45629 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,432 DEBUG [RS:0;c302dddf724e:45629 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,432 DEBUG [RS:0;c302dddf724e:45629 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,432 DEBUG [RS:0;c302dddf724e:45629 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,432 DEBUG [RS:0;c302dddf724e:45629 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,432 DEBUG [RS:0;c302dddf724e:45629 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:16,432 DEBUG [RS:0;c302dddf724e:45629 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c302dddf724e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T00:48:16,432 DEBUG [RS:0;c302dddf724e:45629 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c302dddf724e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T00:48:16,434 INFO [RS:0;c302dddf724e:45629 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,435 INFO [RS:0;c302dddf724e:45629 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,435 INFO [RS:0;c302dddf724e:45629 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,435 INFO [RS:0;c302dddf724e:45629 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,435 INFO [RS:0;c302dddf724e:45629 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,435 INFO [RS:0;c302dddf724e:45629 {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,45629,1731026895993-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T00:48:16,447 INFO [RS:1;c302dddf724e:44739 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T00:48:16,447 INFO [RS:1;c302dddf724e:44739 {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,44739,1731026896029-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,448 INFO [RS:1;c302dddf724e:44739 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,448 INFO [RS:1;c302dddf724e:44739 {}] regionserver.Replication(171): c302dddf724e,44739,1731026896029 started 2024-11-08T00:48:16,453 INFO [RS:0;c302dddf724e:45629 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T00:48:16,453 INFO [RS:0;c302dddf724e:45629 {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,45629,1731026895993-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,453 INFO [RS:0;c302dddf724e:45629 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,453 INFO [RS:0;c302dddf724e:45629 {}] regionserver.Replication(171): c302dddf724e,45629,1731026895993 started 2024-11-08T00:48:16,464 INFO [RS:1;c302dddf724e:44739 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,464 INFO [RS:1;c302dddf724e:44739 {}] regionserver.HRegionServer(1482): Serving as c302dddf724e,44739,1731026896029, RpcServer on c302dddf724e/172.17.0.2:44739, sessionid=0x100f50c2eb70002 2024-11-08T00:48:16,464 DEBUG [RS:1;c302dddf724e:44739 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T00:48:16,464 DEBUG [RS:1;c302dddf724e:44739 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c302dddf724e,44739,1731026896029 2024-11-08T00:48:16,464 DEBUG [RS:1;c302dddf724e:44739 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c302dddf724e,44739,1731026896029' 2024-11-08T00:48:16,464 DEBUG [RS:1;c302dddf724e:44739 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T00:48:16,465 DEBUG [RS:1;c302dddf724e:44739 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T00:48:16,466 DEBUG [RS:1;c302dddf724e:44739 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T00:48:16,466 DEBUG [RS:1;c302dddf724e:44739 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T00:48:16,466 DEBUG [RS:1;c302dddf724e:44739 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c302dddf724e,44739,1731026896029 2024-11-08T00:48:16,466 DEBUG [RS:1;c302dddf724e:44739 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c302dddf724e,44739,1731026896029' 2024-11-08T00:48:16,466 DEBUG [RS:1;c302dddf724e:44739 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T00:48:16,466 DEBUG [RS:1;c302dddf724e:44739 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T00:48:16,467 DEBUG [RS:1;c302dddf724e:44739 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T00:48:16,467 INFO [RS:1;c302dddf724e:44739 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T00:48:16,467 INFO [RS:1;c302dddf724e:44739 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T00:48:16,471 INFO [RS:0;c302dddf724e:45629 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:16,471 INFO [RS:0;c302dddf724e:45629 {}] regionserver.HRegionServer(1482): Serving as c302dddf724e,45629,1731026895993, RpcServer on c302dddf724e/172.17.0.2:45629, sessionid=0x100f50c2eb70001 2024-11-08T00:48:16,472 DEBUG [RS:0;c302dddf724e:45629 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T00:48:16,472 DEBUG [RS:0;c302dddf724e:45629 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c302dddf724e,45629,1731026895993 2024-11-08T00:48:16,472 DEBUG [RS:0;c302dddf724e:45629 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c302dddf724e,45629,1731026895993' 2024-11-08T00:48:16,472 DEBUG [RS:0;c302dddf724e:45629 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T00:48:16,472 DEBUG [RS:0;c302dddf724e:45629 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T00:48:16,473 DEBUG [RS:0;c302dddf724e:45629 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T00:48:16,473 DEBUG [RS:0;c302dddf724e:45629 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T00:48:16,473 DEBUG [RS:0;c302dddf724e:45629 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c302dddf724e,45629,1731026895993 2024-11-08T00:48:16,473 DEBUG [RS:0;c302dddf724e:45629 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c302dddf724e,45629,1731026895993' 2024-11-08T00:48:16,473 DEBUG [RS:0;c302dddf724e:45629 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T00:48:16,474 DEBUG [RS:0;c302dddf724e:45629 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T00:48:16,474 DEBUG [RS:0;c302dddf724e:45629 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T00:48:16,474 INFO [RS:0;c302dddf724e:45629 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T00:48:16,475 INFO [RS:0;c302dddf724e:45629 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T00:48:16,556 WARN [c302dddf724e:36775 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T00:48:16,568 INFO [RS:1;c302dddf724e:44739 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T00:48:16,569 INFO [RS:1;c302dddf724e:44739 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c302dddf724e%2C44739%2C1731026896029, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,44739,1731026896029, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/oldWALs, maxLogs=55 2024-11-08T00:48:16,575 INFO [RS:0;c302dddf724e:45629 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T00:48:16,576 INFO [RS:0;c302dddf724e:45629 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c302dddf724e%2C45629%2C1731026895993, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,45629,1731026895993, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/oldWALs, maxLogs=55 2024-11-08T00:48:16,590 INFO [RS:1;c302dddf724e:44739 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,44739,1731026896029/c302dddf724e%2C44739%2C1731026896029.1731026896571 2024-11-08T00:48:16,590 DEBUG [RS:1;c302dddf724e:44739 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T00:48:16,595 INFO [RS:0;c302dddf724e:45629 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,45629,1731026895993/c302dddf724e%2C45629%2C1731026895993.1731026896577 2024-11-08T00:48:16,595 DEBUG [RS:0;c302dddf724e:45629 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T00:48:16,807 DEBUG [c302dddf724e:36775 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-08T00:48:16,807 DEBUG [c302dddf724e:36775 {}] balancer.BalancerClusterState(204): Hosts are {c302dddf724e=0} racks are {/default-rack=0} 2024-11-08T00:48:16,808 DEBUG [c302dddf724e:36775 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T00:48:16,808 DEBUG [c302dddf724e:36775 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T00:48:16,808 DEBUG [c302dddf724e:36775 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T00:48:16,808 DEBUG [c302dddf724e:36775 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T00:48:16,808 INFO [c302dddf724e:36775 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T00:48:16,808 INFO [c302dddf724e:36775 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T00:48:16,809 DEBUG [c302dddf724e:36775 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T00:48:16,809 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c302dddf724e,45629,1731026895993 2024-11-08T00:48:16,811 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,45629,1731026895993, state=OPENING 2024-11-08T00:48:16,813 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T00:48:16,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44739-0x100f50c2eb70002, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:16,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:16,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:16,819 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:48:16,819 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:48:16,819 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:48:16,819 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:48:16,819 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c302dddf724e,45629,1731026895993}] 2024-11-08T00:48:16,973 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T00:48:16,976 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43611, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T00:48:16,978 INFO [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T00:48:16,978 INFO [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T00:48:16,978 INFO [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T00:48:16,979 INFO [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c302dddf724e%2C45629%2C1731026895993.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,45629,1731026895993, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/oldWALs, maxLogs=55 2024-11-08T00:48:16,999 INFO [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,45629,1731026895993/c302dddf724e%2C45629%2C1731026895993.meta.1731026896980.meta 2024-11-08T00:48:17,000 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T00:48:17,000 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:48:17,000 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T00:48:17,001 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T00:48:17,001 INFO [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T00:48:17,001 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T00:48:17,001 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:48:17,001 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T00:48:17,002 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T00:48:17,003 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T00:48:17,004 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T00:48:17,004 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:17,004 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:48:17,004 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T00:48:17,005 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T00:48:17,005 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:17,005 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:48:17,005 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T00:48:17,006 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T00:48:17,006 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:17,007 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:48:17,007 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T00:48:17,008 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T00:48:17,008 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:17,008 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:48:17,008 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T00:48:17,009 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740 2024-11-08T00:48:17,010 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740 2024-11-08T00:48:17,010 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T00:48:17,010 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T00:48:17,010 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T00:48:17,012 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T00:48:17,012 INFO [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70276021, jitterRate=0.04719431698322296}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T00:48:17,012 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T00:48:17,013 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731026897002Writing region info on filesystem at 1731026897002Initializing all the Stores at 1731026897002Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026897002Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026897002Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026897002Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026897003 (+1 ms)Cleaning up temporary data from old regions at 1731026897010 (+7 ms)Running coprocessor post-open hooks at 1731026897012 (+2 ms)Region opened successfully at 1731026897013 (+1 ms) 2024-11-08T00:48:17,013 INFO [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731026896972 2024-11-08T00:48:17,017 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T00:48:17,017 INFO [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T00:48:17,017 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c302dddf724e,45629,1731026895993 2024-11-08T00:48:17,018 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,45629,1731026895993, state=OPEN 2024-11-08T00:48:17,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:48:17,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:48:17,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44739-0x100f50c2eb70002, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:48:17,020 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:48:17,020 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:48:17,020 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:48:17,020 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c302dddf724e,45629,1731026895993 2024-11-08T00:48:17,023 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T00:48:17,024 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c302dddf724e,45629,1731026895993 in 201 msec 2024-11-08T00:48:17,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T00:48:17,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 622 msec 2024-11-08T00:48:17,027 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T00:48:17,027 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T00:48:17,031 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:48:17,031 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c302dddf724e,45629,1731026895993, seqNum=-1] 2024-11-08T00:48:17,031 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:48:17,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47967, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:48:17,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 728 msec 2024-11-08T00:48:17,040 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731026897040, completionTime=-1 2024-11-08T00:48:17,040 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-08T00:48:17,040 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T00:48:17,043 INFO [master/c302dddf724e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-08T00:48:17,043 INFO [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026957043 2024-11-08T00:48:17,043 INFO [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731027017043 2024-11-08T00:48:17,043 INFO [master/c302dddf724e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-08T00:48:17,043 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,36775,1731026895930-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:17,044 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,36775,1731026895930-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:17,044 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,36775,1731026895930-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:17,044 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c302dddf724e:36775, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:17,044 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:17,044 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:17,046 DEBUG [master/c302dddf724e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T00:48:17,050 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.961sec 2024-11-08T00:48:17,050 INFO [master/c302dddf724e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T00:48:17,050 INFO [master/c302dddf724e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T00:48:17,050 INFO [master/c302dddf724e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T00:48:17,050 INFO [master/c302dddf724e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T00:48:17,050 INFO [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T00:48:17,050 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,36775,1731026895930-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T00:48:17,050 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,36775,1731026895930-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T00:48:17,051 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T00:48:17,051 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T00:48:17,051 INFO [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,36775,1731026895930-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:17,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34126543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:48:17,059 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c302dddf724e,36775,-1 for getting cluster id 2024-11-08T00:48:17,059 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:48:17,062 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '339b3297-2b2f-4d2d-b7ca-a35a4fada4b1' 2024-11-08T00:48:17,062 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:48:17,062 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "339b3297-2b2f-4d2d-b7ca-a35a4fada4b1" 2024-11-08T00:48:17,063 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@710b8f4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:48:17,063 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c302dddf724e,36775,-1] 2024-11-08T00:48:17,063 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:48:17,064 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:48:17,065 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:48:17,066 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46dfee43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:48:17,067 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:48:17,068 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c302dddf724e,45629,1731026895993, seqNum=-1] 2024-11-08T00:48:17,069 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:48:17,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:48:17,073 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c302dddf724e,36775,1731026895930 2024-11-08T00:48:17,074 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T00:48:17,075 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c302dddf724e,36775,1731026895930 2024-11-08T00:48:17,075 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@412c739c 2024-11-08T00:48:17,075 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T00:48:17,077 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T00:48:17,078 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36775 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731026897073', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T00:48:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36775 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731026897073 2024-11-08T00:48:17,080 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731026897073 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T00:48:17,081 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:17,081 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36775 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731026897073" procId is: 4 2024-11-08T00:48:17,082 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731026897073 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T00:48:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T00:48:17,094 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731026897073-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5f10d1fefbc8f1be64e21e67309c4b3a, NAME => 'testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731026897073', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77 2024-11-08T00:48:17,102 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731026897073-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:48:17,102 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731026897073-pool-0 {}] regionserver.HRegion(1722): Closing 5f10d1fefbc8f1be64e21e67309c4b3a, disabling compactions & flushes 2024-11-08T00:48:17,102 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731026897073-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. 2024-11-08T00:48:17,102 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731026897073-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. 2024-11-08T00:48:17,102 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731026897073-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. after waiting 0 ms 2024-11-08T00:48:17,102 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731026897073-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. 2024-11-08T00:48:17,102 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731026897073-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. 2024-11-08T00:48:17,102 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731026897073-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5f10d1fefbc8f1be64e21e67309c4b3a: Waiting for close lock at 1731026897102Disabling compacts and flushes for region at 1731026897102Disabling writes for close at 1731026897102Writing region close event to WAL at 1731026897102Closed at 1731026897102 2024-11-08T00:48:17,103 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731026897073 execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T00:48:17,104 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731026897103"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731026897103"}]},"ts":"1731026897103"} 2024-11-08T00:48:17,107 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T00:48:17,108 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731026897073 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T00:48:17,108 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731026897073","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731026897108"}]},"ts":"1731026897108"} 2024-11-08T00:48:17,111 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731026897073, state=ENABLING in hbase:meta 2024-11-08T00:48:17,111 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {c302dddf724e=0} racks are {/default-rack=0} 2024-11-08T00:48:17,112 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T00:48:17,112 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T00:48:17,112 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T00:48:17,112 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T00:48:17,112 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T00:48:17,112 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T00:48:17,112 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T00:48:17,113 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731026897073, region=5f10d1fefbc8f1be64e21e67309c4b3a, ASSIGN}] 2024-11-08T00:48:17,114 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731026897073, region=5f10d1fefbc8f1be64e21e67309c4b3a, ASSIGN 2024-11-08T00:48:17,115 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731026897073, region=5f10d1fefbc8f1be64e21e67309c4b3a, ASSIGN; state=OFFLINE, location=c302dddf724e,44739,1731026896029; forceNewPlan=false, retain=false 2024-11-08T00:48:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T00:48:17,265 INFO [c302dddf724e:36775 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-08T00:48:17,266 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5f10d1fefbc8f1be64e21e67309c4b3a, regionState=OPENING, regionLocation=c302dddf724e,44739,1731026896029 2024-11-08T00:48:17,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731026897073, region=5f10d1fefbc8f1be64e21e67309c4b3a, ASSIGN because future has completed 2024-11-08T00:48:17,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f10d1fefbc8f1be64e21e67309c4b3a, server=c302dddf724e,44739,1731026896029}] 2024-11-08T00:48:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T00:48:17,423 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T00:48:17,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60221, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T00:48:17,427 INFO [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. 2024-11-08T00:48:17,428 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f10d1fefbc8f1be64e21e67309c4b3a, NAME => 'testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a.', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:48:17,428 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731026897073 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:17,428 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:48:17,429 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:17,429 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:17,430 INFO [StoreOpener-5f10d1fefbc8f1be64e21e67309c4b3a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:17,435 INFO [StoreOpener-5f10d1fefbc8f1be64e21e67309c4b3a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f10d1fefbc8f1be64e21e67309c4b3a columnFamilyName family 2024-11-08T00:48:17,435 DEBUG [StoreOpener-5f10d1fefbc8f1be64e21e67309c4b3a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:17,435 INFO [StoreOpener-5f10d1fefbc8f1be64e21e67309c4b3a-1 {}] regionserver.HStore(327): Store=5f10d1fefbc8f1be64e21e67309c4b3a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:48:17,436 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:17,437 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:17,437 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:17,437 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:17,437 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:17,438 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:17,447 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:48:17,448 INFO [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5f10d1fefbc8f1be64e21e67309c4b3a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62419180, jitterRate=-0.06988173723220825}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:48:17,448 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:17,448 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5f10d1fefbc8f1be64e21e67309c4b3a: Running coprocessor pre-open hook at 1731026897429Writing region info on filesystem at 1731026897429Initializing all the Stores at 1731026897429Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026897429Cleaning up temporary data from old regions at 1731026897437 (+8 ms)Running coprocessor post-open hooks at 1731026897448 (+11 ms)Region opened successfully at 1731026897448 2024-11-08T00:48:17,449 INFO [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a., pid=6, masterSystemTime=1731026897423 2024-11-08T00:48:17,452 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. 2024-11-08T00:48:17,452 INFO [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. 2024-11-08T00:48:17,453 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5f10d1fefbc8f1be64e21e67309c4b3a, regionState=OPEN, openSeqNum=2, regionLocation=c302dddf724e,44739,1731026896029 2024-11-08T00:48:17,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f10d1fefbc8f1be64e21e67309c4b3a, server=c302dddf724e,44739,1731026896029 because future has completed 2024-11-08T00:48:17,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T00:48:17,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5f10d1fefbc8f1be64e21e67309c4b3a, server=c302dddf724e,44739,1731026896029 in 186 msec 2024-11-08T00:48:17,460 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T00:48:17,460 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731026897073, region=5f10d1fefbc8f1be64e21e67309c4b3a, ASSIGN in 345 msec 2024-11-08T00:48:17,461 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731026897073 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T00:48:17,462 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731026897073","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731026897461"}]},"ts":"1731026897461"} 2024-11-08T00:48:17,464 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731026897073, state=ENABLED in hbase:meta 2024-11-08T00:48:17,465 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731026897073 execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T00:48:17,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731026897073 in 387 msec 2024-11-08T00:48:17,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T00:48:17,714 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731026897073 completed 2024-11-08T00:48:17,714 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731026897073 get assigned. Timeout = 60000ms 2024-11-08T00:48:17,715 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T00:48:17,719 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731026897073 assigned to meta. Checking AM states. 2024-11-08T00:48:17,719 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T00:48:17,719 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731026897073 assigned. 2024-11-08T00:48:17,728 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731026897073', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a., hostname=c302dddf724e,44739,1731026896029, seqNum=2] 2024-11-08T00:48:17,729 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:48:17,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:48:17,740 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5f10d1fefbc8f1be64e21e67309c4b3a 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-08T00:48:17,759 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on df45da4f24934b08b9093c220e610e72 2024-11-08T00:48:17,777 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a/.tmp/family/df45da4f24934b08b9093c220e610e72 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-08T00:48:17,778 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a/.tmp/family/df45da4f24934b08b9093c220e610e72 2024-11-08T00:48:17,783 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a/.tmp/family/df45da4f24934b08b9093c220e610e72 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a/family/df45da4f24934b08b9093c220e610e72 2024-11-08T00:48:17,784 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a/family/df45da4f24934b08b9093c220e610e72, delay=922 ms 2024-11-08T00:48:17,786 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a/family/df45da4f24934b08b9093c220e610e72, entries=2, sequenceid=6, filesize=4.9 K 2024-11-08T00:48:17,787 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 5f10d1fefbc8f1be64e21e67309c4b3a in 47ms, sequenceid=6, compaction requested=false 2024-11-08T00:48:17,787 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731026897073' 2024-11-08T00:48:17,788 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5f10d1fefbc8f1be64e21e67309c4b3a: 2024-11-08T00:48:18,177 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T00:48:18,707 DEBUG [hfile-prefetch-1731026895366 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key df45da4f24934b08b9093c220e610e72_0. 2024-11-08T00:48:18,708 DEBUG [hfile-prefetch-1731026895366 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key df45da4f24934b08b9093c220e610e72_117. 2024-11-08T00:48:18,708 DEBUG [hfile-prefetch-1731026895366 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a/family/df45da4f24934b08b9093c220e610e72, with total blocks 2, and data blocks 1 2024-11-08T00:48:18,708 DEBUG [hfile-prefetch-1731026895366 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-08T00:48:18,709 DEBUG [hfile-prefetch-1731026895366 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a/family/df45da4f24934b08b9093c220e610e72, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-08T00:48:18,709 DEBUG [hfile-prefetch-1731026895366 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a/family/df45da4f24934b08b9093c220e610e72. Will sleep 100ms and try the verification again. 2024-11-08T00:48:18,743 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-08T00:48:18,743 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-08T00:48:18,743 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T00:48:18,744 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T00:48:18,748 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-08T00:48:18,748 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731026898743 2024-11-08T00:48:18,748 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-08T00:48:18,789 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;c302dddf724e:44739,5,FailOnTimeoutGroup] 2024-11-08T00:48:18,789 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c302dddf724e,44739,1731026896029' ***** 2024-11-08T00:48:18,789 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-08T00:48:18,789 INFO [RS:1;c302dddf724e:44739 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T00:48:18,790 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T00:48:18,790 INFO [RS:1;c302dddf724e:44739 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be782f04-237a-9a39-9d79-99c0a8908307/bucket.persistence 2024-11-08T00:48:18,790 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T00:48:18,790 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T00:48:18,790 INFO [RS:1;c302dddf724e:44739 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T00:48:18,790 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T00:48:18,800 DEBUG [RS:1;c302dddf724e:44739 {}] bucket.BucketCache(1488): Thread RS:1;c302dddf724e:44739 started persisting bucket cache to file 2024-11-08T00:48:18,800 DEBUG [RS:1;c302dddf724e:44739 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T00:48:18,800 DEBUG [RS:1;c302dddf724e:44739 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T00:48:18,804 DEBUG [RS:1;c302dddf724e:44739 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-08T00:48:18,804 DEBUG [RS:1;c302dddf724e:44739 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731026898800 2024-11-08T00:48:18,804 DEBUG [RS:1;c302dddf724e:44739 {}] bucket.BucketCache(1510): Thread RS:1;c302dddf724e:44739 finished persisting bucket cache to file, renaming 2024-11-08T00:48:18,805 INFO [RS:1;c302dddf724e:44739 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T00:48:18,805 INFO [RS:1;c302dddf724e:44739 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T00:48:18,805 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T00:48:18,805 INFO [RS:1;c302dddf724e:44739 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T00:48:18,805 INFO [RS:1;c302dddf724e:44739 {}] regionserver.HRegionServer(3091): Received CLOSE for 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:18,805 INFO [RS:1;c302dddf724e:44739 {}] regionserver.HRegionServer(959): stopping server c302dddf724e,44739,1731026896029 2024-11-08T00:48:18,805 INFO [RS:1;c302dddf724e:44739 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T00:48:18,805 INFO [RS:1;c302dddf724e:44739 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;c302dddf724e:44739. 2024-11-08T00:48:18,805 INFO [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T00:48:18,806 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5f10d1fefbc8f1be64e21e67309c4b3a, disabling compactions & flushes 2024-11-08T00:48:18,806 DEBUG [RS:1;c302dddf724e:44739 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:48:18,806 INFO [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. 2024-11-08T00:48:18,806 DEBUG [RS:1;c302dddf724e:44739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:48:18,806 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. 2024-11-08T00:48:18,806 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. after waiting 0 ms 2024-11-08T00:48:18,806 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. 2024-11-08T00:48:18,806 INFO [RS:1;c302dddf724e:44739 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T00:48:18,806 DEBUG [RS:1;c302dddf724e:44739 {}] regionserver.HRegionServer(1325): Online Regions={5f10d1fefbc8f1be64e21e67309c4b3a=testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a.} 2024-11-08T00:48:18,806 DEBUG [RS:1;c302dddf724e:44739 {}] regionserver.HRegionServer(1351): Waiting on 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:18,809 DEBUG [hfile-prefetch-1731026895366 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a/family/df45da4f24934b08b9093c220e610e72, with total blocks 2, and data blocks 1 2024-11-08T00:48:18,809 DEBUG [hfile-prefetch-1731026895366 {}] bucket.BucketCache(2286): iterating over 1 entries in the backing map 2024-11-08T00:48:18,809 DEBUG [hfile-prefetch-1731026895366 {}] bucket.BucketCache(2294): found block for file df45da4f24934b08b9093c220e610e72 in the backing map. Acquiring read lock for offset 0 2024-11-08T00:48:18,809 DEBUG [hfile-prefetch-1731026895366 {}] bucket.BucketCache(2306): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a/family/df45da4f24934b08b9093c220e610e72 has now been fully cached. 2024-11-08T00:48:18,809 DEBUG [StoreFileCloser-5f10d1fefbc8f1be64e21e67309c4b3a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a/family/df45da4f24934b08b9093c220e610e72 2024-11-08T00:48:18,809 DEBUG [hfile-prefetch-1731026895366 {}] hfile.PrefetchExecutor(121): Prefetch completed for df45da4f24934b08b9093c220e610e72 2024-11-08T00:48:18,815 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-08T00:48:18,816 INFO [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. 2024-11-08T00:48:18,816 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5f10d1fefbc8f1be64e21e67309c4b3a: Waiting for close lock at 1731026898805Running coprocessor pre-close hooks at 1731026898806 (+1 ms)Disabling compacts and flushes for region at 1731026898806Disabling writes for close at 1731026898806Writing region close event to WAL at 1731026898809 (+3 ms)Running coprocessor post-close hooks at 1731026898816 (+7 ms)Closed at 1731026898816 2024-11-08T00:48:18,816 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. 2024-11-08T00:48:18,831 INFO [regionserver/c302dddf724e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T00:48:19,006 INFO [RS:1;c302dddf724e:44739 {}] regionserver.HRegionServer(976): stopping server c302dddf724e,44739,1731026896029; all regions closed. 2024-11-08T00:48:19,009 DEBUG [RS:1;c302dddf724e:44739 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/oldWALs 2024-11-08T00:48:19,009 INFO [RS:1;c302dddf724e:44739 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c302dddf724e%2C44739%2C1731026896029:(num 1731026896571) 2024-11-08T00:48:19,009 DEBUG [RS:1;c302dddf724e:44739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:48:19,009 INFO [RS:1;c302dddf724e:44739 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T00:48:19,009 INFO [RS:1;c302dddf724e:44739 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T00:48:19,010 INFO [RS:1;c302dddf724e:44739 {}] hbase.ChoreService(370): Chore service for: regionserver/c302dddf724e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T00:48:19,010 INFO [RS:1;c302dddf724e:44739 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T00:48:19,010 INFO [regionserver/c302dddf724e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T00:48:19,010 INFO [RS:1;c302dddf724e:44739 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T00:48:19,010 INFO [RS:1;c302dddf724e:44739 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T00:48:19,010 INFO [RS:1;c302dddf724e:44739 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T00:48:19,010 INFO [RS:1;c302dddf724e:44739 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44739 2024-11-08T00:48:19,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44739-0x100f50c2eb70002, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c302dddf724e,44739,1731026896029 2024-11-08T00:48:19,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T00:48:19,016 INFO [RS:1;c302dddf724e:44739 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T00:48:19,018 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c302dddf724e,44739,1731026896029] 2024-11-08T00:48:19,020 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c302dddf724e,44739,1731026896029 already deleted, retry=false 2024-11-08T00:48:19,020 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of c302dddf724e,44739,1731026896029 on c302dddf724e,36775,1731026895930 2024-11-08T00:48:19,025 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure c302dddf724e,44739,1731026896029, splitWal=true, meta=false 2024-11-08T00:48:19,027 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for c302dddf724e,44739,1731026896029 (carryingMeta=false) c302dddf724e,44739,1731026896029/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@44a28f39[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-08T00:48:19,027 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure c302dddf724e,44739,1731026896029, splitWal=true, meta=false 2024-11-08T00:48:19,029 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): c302dddf724e,44739,1731026896029 had 1 regions 2024-11-08T00:48:19,030 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure c302dddf724e,44739,1731026896029, splitWal=true, meta=false, isMeta: false 2024-11-08T00:48:19,031 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,44739,1731026896029-splitting 2024-11-08T00:48:19,031 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,44739,1731026896029-splitting dir is empty, no logs to split. 2024-11-08T00:48:19,031 INFO [PEWorker-2 {}] master.SplitWALManager(105): c302dddf724e,44739,1731026896029 WAL count=0, meta=false 2024-11-08T00:48:19,033 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,44739,1731026896029-splitting dir is empty, no logs to split. 2024-11-08T00:48:19,033 INFO [PEWorker-2 {}] master.SplitWALManager(105): c302dddf724e,44739,1731026896029 WAL count=0, meta=false 2024-11-08T00:48:19,033 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if c302dddf724e,44739,1731026896029 WAL splitting is done? wals=0, meta=false 2024-11-08T00:48:19,034 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for c302dddf724e,44739,1731026896029 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,44739,1731026896029-splitting does not exist 2024-11-08T00:48:19,036 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731026897073, region=5f10d1fefbc8f1be64e21e67309c4b3a, ASSIGN}] 2024-11-08T00:48:19,037 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731026897073, region=5f10d1fefbc8f1be64e21e67309c4b3a, ASSIGN 2024-11-08T00:48:19,038 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731026897073, region=5f10d1fefbc8f1be64e21e67309c4b3a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T00:48:19,118 INFO [RS:1;c302dddf724e:44739 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T00:48:19,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44739-0x100f50c2eb70002, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:48:19,118 INFO [RS:1;c302dddf724e:44739 {}] regionserver.HRegionServer(1031): Exiting; stopping=c302dddf724e,44739,1731026896029; zookeeper connection closed. 2024-11-08T00:48:19,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44739-0x100f50c2eb70002, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:48:19,119 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3f95a001 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3f95a001 2024-11-08T00:48:19,189 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=5f10d1fefbc8f1be64e21e67309c4b3a, regionState=OPENING, regionLocation=c302dddf724e,45629,1731026895993 2024-11-08T00:48:19,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731026897073, region=5f10d1fefbc8f1be64e21e67309c4b3a, ASSIGN because future has completed 2024-11-08T00:48:19,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f10d1fefbc8f1be64e21e67309c4b3a, server=c302dddf724e,45629,1731026895993}] 2024-11-08T00:48:19,306 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c302dddf724e:0 server-side Connection retries=45 2024-11-08T00:48:19,306 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:19,306 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:19,306 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T00:48:19,306 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:48:19,306 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T00:48:19,306 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T00:48:19,306 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T00:48:19,307 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37111 2024-11-08T00:48:19,309 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37111 connecting to ZooKeeper ensemble=127.0.0.1:58229 2024-11-08T00:48:19,309 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:19,309 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:19,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:371110x0, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T00:48:19,313 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37111-0x100f50c2eb70003 connected 2024-11-08T00:48:19,313 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:371110x0, quorum=127.0.0.1:58229, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-08T00:48:19,313 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-08T00:48:19,314 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T00:48:19,314 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be782f04-237a-9a39-9d79-99c0a8908307/bucket.cache 2024-11-08T00:48:19,314 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T00:48:19,314 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T00:48:19,314 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T00:48:19,315 INFO [Thread-200 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T00:48:19,315 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be782f04-237a-9a39-9d79-99c0a8908307/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be782f04-237a-9a39-9d79-99c0a8908307/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T00:48:19,315 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T00:48:19,316 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37111-0x100f50c2eb70003, quorum=127.0.0.1:58229, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T00:48:19,316 INFO [Thread-200 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-08T00:48:19,317 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37111-0x100f50c2eb70003, quorum=127.0.0.1:58229, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T00:48:19,317 INFO [Thread-200 {}] bucket.BucketCache(1706): Retrieved 1 of chunks with blockCount = 1. 2024-11-08T00:48:19,317 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37111 2024-11-08T00:48:19,318 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37111 2024-11-08T00:48:19,318 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37111 2024-11-08T00:48:19,318 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37111 2024-11-08T00:48:19,319 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37111 2024-11-08T00:48:19,320 INFO [RS:2;c302dddf724e:37111 {}] regionserver.HRegionServer(746): ClusterId : 339b3297-2b2f-4d2d-b7ca-a35a4fada4b1 2024-11-08T00:48:19,320 DEBUG [RS:2;c302dddf724e:37111 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T00:48:19,320 DEBUG [Thread-200 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-08T00:48:19,321 INFO [Thread-200 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T00:48:19,321 INFO [Thread-200 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 1 2024-11-08T00:48:19,321 INFO [Thread-200 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be782f04-237a-9a39-9d79-99c0a8908307/bucket.persistence is complete. 2024-11-08T00:48:19,323 DEBUG [RS:2;c302dddf724e:37111 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T00:48:19,323 DEBUG [RS:2;c302dddf724e:37111 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T00:48:19,326 DEBUG [RS:2;c302dddf724e:37111 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T00:48:19,326 DEBUG [RS:2;c302dddf724e:37111 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@635fba9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c302dddf724e/172.17.0.2:0 2024-11-08T00:48:19,338 DEBUG [RS:2;c302dddf724e:37111 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;c302dddf724e:37111 2024-11-08T00:48:19,338 INFO [RS:2;c302dddf724e:37111 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T00:48:19,338 INFO [RS:2;c302dddf724e:37111 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T00:48:19,338 DEBUG [RS:2;c302dddf724e:37111 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T00:48:19,339 INFO [RS:2;c302dddf724e:37111 {}] regionserver.HRegionServer(2659): reportForDuty to master=c302dddf724e,36775,1731026895930 with port=37111, startcode=1731026899305 2024-11-08T00:48:19,339 DEBUG [RS:2;c302dddf724e:37111 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T00:48:19,341 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52231, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T00:48:19,342 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36775 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c302dddf724e,37111,1731026899305 2024-11-08T00:48:19,342 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36775 {}] master.ServerManager(517): Registering regionserver=c302dddf724e,37111,1731026899305 2024-11-08T00:48:19,343 DEBUG [RS:2;c302dddf724e:37111 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77 2024-11-08T00:48:19,343 DEBUG [RS:2;c302dddf724e:37111 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T00:48:19,343 DEBUG [RS:2;c302dddf724e:37111 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T00:48:19,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T00:48:19,346 DEBUG [RS:2;c302dddf724e:37111 {}] zookeeper.ZKUtil(111): regionserver:37111-0x100f50c2eb70003, quorum=127.0.0.1:58229, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c302dddf724e,37111,1731026899305 2024-11-08T00:48:19,346 WARN [RS:2;c302dddf724e:37111 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T00:48:19,346 INFO [RS:2;c302dddf724e:37111 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T00:48:19,346 DEBUG [RS:2;c302dddf724e:37111 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,37111,1731026899305 2024-11-08T00:48:19,346 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c302dddf724e,37111,1731026899305] 2024-11-08T00:48:19,348 INFO [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. 2024-11-08T00:48:19,348 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f10d1fefbc8f1be64e21e67309c4b3a, NAME => 'testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a.', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:48:19,349 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731026897073 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:19,349 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:48:19,349 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:19,349 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:19,351 INFO [StoreOpener-5f10d1fefbc8f1be64e21e67309c4b3a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:19,352 INFO [StoreOpener-5f10d1fefbc8f1be64e21e67309c4b3a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f10d1fefbc8f1be64e21e67309c4b3a columnFamilyName family 2024-11-08T00:48:19,352 DEBUG [StoreOpener-5f10d1fefbc8f1be64e21e67309c4b3a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:48:19,353 INFO [RS:2;c302dddf724e:37111 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T00:48:19,355 DEBUG [StoreFileOpener-5f10d1fefbc8f1be64e21e67309c4b3a-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a/family/df45da4f24934b08b9093c220e610e72, delay=933 ms 2024-11-08T00:48:19,356 DEBUG [StoreOpener-5f10d1fefbc8f1be64e21e67309c4b3a-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a/family/df45da4f24934b08b9093c220e610e72 2024-11-08T00:48:19,356 INFO [StoreOpener-5f10d1fefbc8f1be64e21e67309c4b3a-1 {}] regionserver.HStore(327): Store=5f10d1fefbc8f1be64e21e67309c4b3a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:48:19,356 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:19,357 INFO [RS:2;c302dddf724e:37111 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T00:48:19,357 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:19,357 INFO [RS:2;c302dddf724e:37111 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T00:48:19,357 INFO [RS:2;c302dddf724e:37111 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:19,358 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:19,358 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:19,358 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:19,359 INFO [RS:2;c302dddf724e:37111 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T00:48:19,360 INFO [RS:2;c302dddf724e:37111 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T00:48:19,360 INFO [RS:2;c302dddf724e:37111 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:19,360 DEBUG [RS:2;c302dddf724e:37111 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:19,360 DEBUG [RS:2;c302dddf724e:37111 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:19,360 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:19,360 DEBUG [RS:2;c302dddf724e:37111 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:19,360 DEBUG [RS:2;c302dddf724e:37111 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:19,360 DEBUG [RS:2;c302dddf724e:37111 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:19,360 DEBUG [RS:2;c302dddf724e:37111 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c302dddf724e:0, corePoolSize=2, maxPoolSize=2 2024-11-08T00:48:19,360 DEBUG [RS:2;c302dddf724e:37111 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:19,360 DEBUG [RS:2;c302dddf724e:37111 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:19,360 DEBUG [RS:2;c302dddf724e:37111 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:19,361 DEBUG [RS:2;c302dddf724e:37111 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:19,361 DEBUG [RS:2;c302dddf724e:37111 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:19,361 DEBUG [RS:2;c302dddf724e:37111 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:48:19,361 DEBUG [RS:2;c302dddf724e:37111 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c302dddf724e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T00:48:19,361 DEBUG [RS:2;c302dddf724e:37111 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c302dddf724e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T00:48:19,361 INFO [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 5f10d1fefbc8f1be64e21e67309c4b3a; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61803771, jitterRate=-0.07905204594135284}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:48:19,361 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:19,361 INFO [RS:2;c302dddf724e:37111 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:19,361 INFO [RS:2;c302dddf724e:37111 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:19,361 INFO [RS:2;c302dddf724e:37111 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:19,361 INFO [RS:2;c302dddf724e:37111 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:19,362 INFO [RS:2;c302dddf724e:37111 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:19,362 INFO [RS:2;c302dddf724e:37111 {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,37111,1731026899305-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T00:48:19,362 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 5f10d1fefbc8f1be64e21e67309c4b3a: Running coprocessor pre-open hook at 1731026899349Writing region info on filesystem at 1731026899349Initializing all the Stores at 1731026899349Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026899350 (+1 ms)Cleaning up temporary data from old regions at 1731026899358 (+8 ms)Running coprocessor post-open hooks at 1731026899361 (+3 ms)Region opened successfully at 1731026899362 (+1 ms) 2024-11-08T00:48:19,363 INFO [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a., pid=9, masterSystemTime=1731026899346 2024-11-08T00:48:19,366 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. 2024-11-08T00:48:19,366 INFO [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. 2024-11-08T00:48:19,366 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=5f10d1fefbc8f1be64e21e67309c4b3a, regionState=OPEN, openSeqNum=10, regionLocation=c302dddf724e,45629,1731026895993 2024-11-08T00:48:19,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f10d1fefbc8f1be64e21e67309c4b3a, server=c302dddf724e,45629,1731026895993 because future has completed 2024-11-08T00:48:19,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-08T00:48:19,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 5f10d1fefbc8f1be64e21e67309c4b3a, server=c302dddf724e,45629,1731026895993 in 176 msec 2024-11-08T00:48:19,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-08T00:48:19,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731026897073, region=5f10d1fefbc8f1be64e21e67309c4b3a, ASSIGN in 336 msec 2024-11-08T00:48:19,373 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server c302dddf724e,44739,1731026896029 after splitting done 2024-11-08T00:48:19,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure c302dddf724e,44739,1731026896029, splitWal=true, meta=false in 352 msec 2024-11-08T00:48:19,388 INFO [RS:2;c302dddf724e:37111 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T00:48:19,388 INFO [RS:2;c302dddf724e:37111 {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,37111,1731026899305-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:19,388 INFO [RS:2;c302dddf724e:37111 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:19,388 INFO [RS:2;c302dddf724e:37111 {}] regionserver.Replication(171): c302dddf724e,37111,1731026899305 started 2024-11-08T00:48:19,403 INFO [RS:2;c302dddf724e:37111 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:48:19,403 INFO [RS:2;c302dddf724e:37111 {}] regionserver.HRegionServer(1482): Serving as c302dddf724e,37111,1731026899305, RpcServer on c302dddf724e/172.17.0.2:37111, sessionid=0x100f50c2eb70003 2024-11-08T00:48:19,403 DEBUG [RS:2;c302dddf724e:37111 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T00:48:19,403 DEBUG [RS:2;c302dddf724e:37111 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c302dddf724e,37111,1731026899305 2024-11-08T00:48:19,403 DEBUG [RS:2;c302dddf724e:37111 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c302dddf724e,37111,1731026899305' 2024-11-08T00:48:19,403 DEBUG [RS:2;c302dddf724e:37111 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T00:48:19,404 DEBUG [RS:2;c302dddf724e:37111 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T00:48:19,405 DEBUG [RS:2;c302dddf724e:37111 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T00:48:19,405 DEBUG [RS:2;c302dddf724e:37111 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T00:48:19,405 DEBUG [RS:2;c302dddf724e:37111 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c302dddf724e,37111,1731026899305 2024-11-08T00:48:19,405 DEBUG [RS:2;c302dddf724e:37111 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c302dddf724e,37111,1731026899305' 2024-11-08T00:48:19,405 DEBUG [RS:2;c302dddf724e:37111 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T00:48:19,405 DEBUG [RS:2;c302dddf724e:37111 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T00:48:19,406 DEBUG [RS:2;c302dddf724e:37111 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T00:48:19,406 INFO [RS:2;c302dddf724e:37111 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T00:48:19,406 INFO [RS:2;c302dddf724e:37111 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T00:48:19,506 INFO [RS:2;c302dddf724e:37111 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T00:48:19,507 INFO [RS:2;c302dddf724e:37111 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c302dddf724e%2C37111%2C1731026899305, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,37111,1731026899305, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/oldWALs, maxLogs=55 2024-11-08T00:48:19,527 INFO [RS:2;c302dddf724e:37111 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/WALs/c302dddf724e,37111,1731026899305/c302dddf724e%2C37111%2C1731026899305.1731026899508 2024-11-08T00:48:19,527 DEBUG [RS:2;c302dddf724e:37111 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T00:48:19,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T00:48:19,830 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-08T00:48:19,832 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1731026897073 2024-11-08T00:48:19,832 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1731026897073 Metrics about Tables on a single HBase RegionServer 2024-11-08T00:48:19,833 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T00:48:19,833 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-08T00:48:19,833 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-08T00:48:19,833 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-08T00:48:19,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T00:48:19,904 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T00:48:19,904 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:48:19,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:48:19,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:48:19,904 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:48:19,904 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T00:48:19,904 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=92348902, stopped=false 2024-11-08T00:48:19,904 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c302dddf724e,36775,1731026895930 2024-11-08T00:48:19,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T00:48:19,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T00:48:19,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:19,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:19,906 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:48:19,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37111-0x100f50c2eb70003, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T00:48:19,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37111-0x100f50c2eb70003, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:19,906 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T00:48:19,907 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:48:19,907 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:48:19,907 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c302dddf724e,45629,1731026895993' ***** 2024-11-08T00:48:19,907 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T00:48:19,907 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c302dddf724e,37111,1731026899305' ***** 2024-11-08T00:48:19,907 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T00:48:19,907 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37111-0x100f50c2eb70003, quorum=127.0.0.1:58229, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:48:19,907 INFO [RS:0;c302dddf724e:45629 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T00:48:19,907 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:48:19,908 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:48:19,908 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T00:48:19,908 INFO [RS:0;c302dddf724e:45629 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be782f04-237a-9a39-9d79-99c0a8908307/bucket.persistence 2024-11-08T00:48:19,908 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T00:48:19,908 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T00:48:19,908 INFO [RS:2;c302dddf724e:37111 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T00:48:19,908 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T00:48:19,908 INFO [RS:0;c302dddf724e:45629 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T00:48:19,908 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T00:48:19,909 INFO [RS:2;c302dddf724e:37111 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be782f04-237a-9a39-9d79-99c0a8908307/bucket.persistence 2024-11-08T00:48:19,909 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T00:48:19,909 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T00:48:19,909 INFO [RS:2;c302dddf724e:37111 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T00:48:19,909 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T00:48:19,919 DEBUG [RS:0;c302dddf724e:45629 {}] bucket.BucketCache(1488): Thread RS:0;c302dddf724e:45629 started persisting bucket cache to file 2024-11-08T00:48:19,919 DEBUG [RS:0;c302dddf724e:45629 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T00:48:19,919 DEBUG [RS:0;c302dddf724e:45629 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T00:48:19,919 DEBUG [RS:2;c302dddf724e:37111 {}] bucket.BucketCache(1488): Thread RS:2;c302dddf724e:37111 started persisting bucket cache to file 2024-11-08T00:48:19,919 DEBUG [RS:2;c302dddf724e:37111 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T00:48:19,919 DEBUG [RS:2;c302dddf724e:37111 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T00:48:19,922 DEBUG [RS:0;c302dddf724e:45629 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T00:48:19,922 DEBUG [RS:0;c302dddf724e:45629 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731026899919 2024-11-08T00:48:19,922 DEBUG [RS:2;c302dddf724e:37111 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-08T00:48:19,922 DEBUG [RS:2;c302dddf724e:37111 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731026899919 2024-11-08T00:48:19,923 DEBUG [RS:2;c302dddf724e:37111 {}] bucket.BucketCache(1510): Thread RS:2;c302dddf724e:37111 finished persisting bucket cache to file, renaming 2024-11-08T00:48:19,923 DEBUG [RS:0;c302dddf724e:45629 {}] bucket.BucketCache(1510): Thread RS:0;c302dddf724e:45629 finished persisting bucket cache to file, renaming 2024-11-08T00:48:19,923 WARN [RS:0;c302dddf724e:45629 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-08T00:48:19,923 INFO [RS:2;c302dddf724e:37111 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T00:48:19,923 INFO [RS:0;c302dddf724e:45629 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T00:48:19,923 INFO [RS:2;c302dddf724e:37111 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T00:48:19,923 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T00:48:19,923 INFO [RS:2;c302dddf724e:37111 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T00:48:19,923 INFO [RS:0;c302dddf724e:45629 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T00:48:19,923 INFO [RS:2;c302dddf724e:37111 {}] regionserver.HRegionServer(959): stopping server c302dddf724e,37111,1731026899305 2024-11-08T00:48:19,923 INFO [RS:0;c302dddf724e:45629 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T00:48:19,923 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T00:48:19,923 INFO [RS:2;c302dddf724e:37111 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T00:48:19,923 INFO [RS:0;c302dddf724e:45629 {}] regionserver.HRegionServer(3091): Received CLOSE for 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:19,923 INFO [RS:2;c302dddf724e:37111 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;c302dddf724e:37111. 2024-11-08T00:48:19,923 DEBUG [RS:2;c302dddf724e:37111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:48:19,923 DEBUG [RS:2;c302dddf724e:37111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:48:19,923 INFO [RS:2;c302dddf724e:37111 {}] regionserver.HRegionServer(976): stopping server c302dddf724e,37111,1731026899305; all regions closed. 2024-11-08T00:48:19,924 INFO [RS:0;c302dddf724e:45629 {}] regionserver.HRegionServer(959): stopping server c302dddf724e,45629,1731026895993 2024-11-08T00:48:19,924 INFO [RS:0;c302dddf724e:45629 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T00:48:19,924 INFO [RS:0;c302dddf724e:45629 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c302dddf724e:45629. 2024-11-08T00:48:19,924 INFO [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T00:48:19,924 DEBUG [RS:0;c302dddf724e:45629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:48:19,924 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5f10d1fefbc8f1be64e21e67309c4b3a, disabling compactions & flushes 2024-11-08T00:48:19,924 DEBUG [RS:0;c302dddf724e:45629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:48:19,924 INFO [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. 2024-11-08T00:48:19,924 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. 2024-11-08T00:48:19,924 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. after waiting 0 ms 2024-11-08T00:48:19,924 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. 2024-11-08T00:48:19,924 INFO [RS:0;c302dddf724e:45629 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T00:48:19,924 INFO [RS:0;c302dddf724e:45629 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T00:48:19,924 INFO [RS:0;c302dddf724e:45629 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T00:48:19,924 INFO [RS:0;c302dddf724e:45629 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T00:48:19,926 INFO [RS:0;c302dddf724e:45629 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-08T00:48:19,926 DEBUG [RS:0;c302dddf724e:45629 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 5f10d1fefbc8f1be64e21e67309c4b3a=testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a.} 2024-11-08T00:48:19,926 DEBUG [RS:0;c302dddf724e:45629 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 5f10d1fefbc8f1be64e21e67309c4b3a 2024-11-08T00:48:19,926 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T00:48:19,926 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T00:48:19,926 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T00:48:19,926 DEBUG [StoreFileCloser-5f10d1fefbc8f1be64e21e67309c4b3a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a/family/df45da4f24934b08b9093c220e610e72 2024-11-08T00:48:19,926 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T00:48:19,926 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T00:48:19,926 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T00:48:19,926 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-08T00:48:19,927 DEBUG [RS:2;c302dddf724e:37111 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/oldWALs 2024-11-08T00:48:19,927 INFO [RS:2;c302dddf724e:37111 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c302dddf724e%2C37111%2C1731026899305:(num 1731026899508) 2024-11-08T00:48:19,927 DEBUG [RS:2;c302dddf724e:37111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:48:19,927 INFO [RS:2;c302dddf724e:37111 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T00:48:19,927 INFO [RS:2;c302dddf724e:37111 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T00:48:19,927 INFO [RS:2;c302dddf724e:37111 {}] hbase.ChoreService(370): Chore service for: regionserver/c302dddf724e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T00:48:19,928 INFO [RS:2;c302dddf724e:37111 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T00:48:19,928 INFO [RS:2;c302dddf724e:37111 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T00:48:19,928 INFO [RS:2;c302dddf724e:37111 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T00:48:19,928 INFO [RS:2;c302dddf724e:37111 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T00:48:19,928 INFO [regionserver/c302dddf724e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T00:48:19,928 INFO [RS:2;c302dddf724e:37111 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37111 2024-11-08T00:48:19,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T00:48:19,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37111-0x100f50c2eb70003, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c302dddf724e,37111,1731026899305 2024-11-08T00:48:19,931 INFO [RS:2;c302dddf724e:37111 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T00:48:19,932 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c302dddf724e,37111,1731026899305] 2024-11-08T00:48:19,933 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c302dddf724e,37111,1731026899305 already deleted, retry=false 2024-11-08T00:48:19,934 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c302dddf724e,37111,1731026899305 expired; onlineServers=1 2024-11-08T00:48:19,936 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/default/testBlockEvictionOnGracefulStop1731026897073/5f10d1fefbc8f1be64e21e67309c4b3a/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-08T00:48:19,936 INFO [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. 2024-11-08T00:48:19,937 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5f10d1fefbc8f1be64e21e67309c4b3a: Waiting for close lock at 1731026899924Running coprocessor pre-close hooks at 1731026899924Disabling compacts and flushes for region at 1731026899924Disabling writes for close at 1731026899924Writing region close event to WAL at 1731026899926 (+2 ms)Running coprocessor post-close hooks at 1731026899936 (+10 ms)Closed at 1731026899936 2024-11-08T00:48:19,937 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a. 2024-11-08T00:48:19,937 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1f2d79a9416b4b7c8f98c1e037276c12 2024-11-08T00:48:19,939 INFO [regionserver/c302dddf724e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T00:48:19,952 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/.tmp/info/1f2d79a9416b4b7c8f98c1e037276c12 is 209, key is testBlockEvictionOnGracefulStop1731026897073,,1731026897078.5f10d1fefbc8f1be64e21e67309c4b3a./info:regioninfo/1731026899366/Put/seqid=0 2024-11-08T00:48:19,953 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/.tmp/info/1f2d79a9416b4b7c8f98c1e037276c12 2024-11-08T00:48:19,962 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 59b822dc9e264890899ab82347079062 2024-11-08T00:48:19,964 INFO [regionserver/c302dddf724e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T00:48:19,976 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/.tmp/ns/59b822dc9e264890899ab82347079062 is 43, key is default/ns:d/1731026897034/Put/seqid=0 2024-11-08T00:48:19,976 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/.tmp/ns/59b822dc9e264890899ab82347079062 2024-11-08T00:48:19,984 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ac3921c78b3d4f3ab8e6f8372fc08807 2024-11-08T00:48:19,997 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/.tmp/table/ac3921c78b3d4f3ab8e6f8372fc08807 is 80, key is testBlockEvictionOnGracefulStop1731026897073/table:state/1731026897461/Put/seqid=0 2024-11-08T00:48:19,998 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/.tmp/table/ac3921c78b3d4f3ab8e6f8372fc08807 2024-11-08T00:48:20,000 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/.tmp/info/1f2d79a9416b4b7c8f98c1e037276c12 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/info/1f2d79a9416b4b7c8f98c1e037276c12 2024-11-08T00:48:20,001 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/info/1f2d79a9416b4b7c8f98c1e037276c12, delay=957 ms 2024-11-08T00:48:20,002 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/info/1f2d79a9416b4b7c8f98c1e037276c12, entries=14, sequenceid=13, filesize=7.5 K 2024-11-08T00:48:20,002 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/.tmp/ns/59b822dc9e264890899ab82347079062 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/ns/59b822dc9e264890899ab82347079062 2024-11-08T00:48:20,003 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/ns/59b822dc9e264890899ab82347079062, delay=990 ms 2024-11-08T00:48:20,004 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/ns/59b822dc9e264890899ab82347079062, entries=2, sequenceid=13, filesize=5.0 K 2024-11-08T00:48:20,005 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/.tmp/table/ac3921c78b3d4f3ab8e6f8372fc08807 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/table/ac3921c78b3d4f3ab8e6f8372fc08807 2024-11-08T00:48:20,006 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/table/ac3921c78b3d4f3ab8e6f8372fc08807, delay=977 ms 2024-11-08T00:48:20,006 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/table/ac3921c78b3d4f3ab8e6f8372fc08807, entries=2, sequenceid=13, filesize=5.3 K 2024-11-08T00:48:20,007 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 81ms, sequenceid=13, compaction requested=false 2024-11-08T00:48:20,008 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/info/1f2d79a9416b4b7c8f98c1e037276c12 2024-11-08T00:48:20,009 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/ns/59b822dc9e264890899ab82347079062 2024-11-08T00:48:20,009 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/table/ac3921c78b3d4f3ab8e6f8372fc08807 2024-11-08T00:48:20,015 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-08T00:48:20,015 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T00:48:20,015 INFO [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T00:48:20,015 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731026899926Running coprocessor pre-close hooks at 1731026899926Disabling compacts and flushes for region at 1731026899926Disabling writes for close at 1731026899926Obtaining lock to block concurrent updates at 1731026899926Preparing flush snapshotting stores in 1588230740 at 1731026899926Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1731026899927 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731026899927Flushing 1588230740/info: creating writer at 1731026899927Flushing 1588230740/info: appending metadata at 1731026899951 (+24 ms)Flushing 1588230740/info: closing flushed file at 1731026899951Flushing 1588230740/ns: creating writer at 1731026899955 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731026899975 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731026899975Flushing 1588230740/table: creating writer at 1731026899978 (+3 ms)Flushing 1588230740/table: appending metadata at 1731026899997 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731026899997Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ddf09f8: reopening flushed file at 1731026900000 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79c71b30: reopening flushed file at 1731026900002 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50e23299: reopening flushed file at 1731026900004 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 81ms, sequenceid=13, compaction requested=false at 1731026900007 (+3 ms)Writing region close event to WAL at 1731026900009 (+2 ms)Running coprocessor post-close hooks at 1731026900015 (+6 ms)Closed at 1731026900015 2024-11-08T00:48:20,015 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T00:48:20,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37111-0x100f50c2eb70003, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:48:20,033 INFO [RS:2;c302dddf724e:37111 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T00:48:20,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37111-0x100f50c2eb70003, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:48:20,033 INFO [RS:2;c302dddf724e:37111 {}] regionserver.HRegionServer(1031): Exiting; stopping=c302dddf724e,37111,1731026899305; zookeeper connection closed. 2024-11-08T00:48:20,033 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@55cbb5fe {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@55cbb5fe 2024-11-08T00:48:20,126 INFO [RS:0;c302dddf724e:45629 {}] regionserver.HRegionServer(976): stopping server c302dddf724e,45629,1731026895993; all regions closed. 2024-11-08T00:48:20,128 DEBUG [RS:0;c302dddf724e:45629 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/oldWALs 2024-11-08T00:48:20,128 INFO [RS:0;c302dddf724e:45629 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c302dddf724e%2C45629%2C1731026895993.meta:.meta(num 1731026896980) 2024-11-08T00:48:20,130 DEBUG [RS:0;c302dddf724e:45629 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/oldWALs 2024-11-08T00:48:20,130 INFO [RS:0;c302dddf724e:45629 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c302dddf724e%2C45629%2C1731026895993:(num 1731026896577) 2024-11-08T00:48:20,130 DEBUG [RS:0;c302dddf724e:45629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:48:20,130 INFO [RS:0;c302dddf724e:45629 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T00:48:20,130 INFO [RS:0;c302dddf724e:45629 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T00:48:20,130 INFO [RS:0;c302dddf724e:45629 {}] hbase.ChoreService(370): Chore service for: regionserver/c302dddf724e:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T00:48:20,131 INFO [RS:0;c302dddf724e:45629 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T00:48:20,131 INFO [regionserver/c302dddf724e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T00:48:20,131 INFO [RS:0;c302dddf724e:45629 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45629 2024-11-08T00:48:20,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c302dddf724e,45629,1731026895993 2024-11-08T00:48:20,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T00:48:20,133 INFO [RS:0;c302dddf724e:45629 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T00:48:20,136 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c302dddf724e,45629,1731026895993] 2024-11-08T00:48:20,137 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c302dddf724e,45629,1731026895993 already deleted, retry=false 2024-11-08T00:48:20,137 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c302dddf724e,45629,1731026895993 expired; onlineServers=0 2024-11-08T00:48:20,138 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c302dddf724e,36775,1731026895930' ***** 2024-11-08T00:48:20,138 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T00:48:20,138 INFO [M:0;c302dddf724e:36775 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T00:48:20,138 INFO [M:0;c302dddf724e:36775 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T00:48:20,138 DEBUG [M:0;c302dddf724e:36775 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T00:48:20,138 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T00:48:20,138 DEBUG [M:0;c302dddf724e:36775 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T00:48:20,138 DEBUG [master/c302dddf724e:0:becomeActiveMaster-HFileCleaner.large.0-1731026896326 {}] cleaner.HFileCleaner(306): Exit Thread[master/c302dddf724e:0:becomeActiveMaster-HFileCleaner.large.0-1731026896326,5,FailOnTimeoutGroup] 2024-11-08T00:48:20,138 DEBUG [master/c302dddf724e:0:becomeActiveMaster-HFileCleaner.small.0-1731026896326 {}] cleaner.HFileCleaner(306): Exit Thread[master/c302dddf724e:0:becomeActiveMaster-HFileCleaner.small.0-1731026896326,5,FailOnTimeoutGroup] 2024-11-08T00:48:20,138 INFO [M:0;c302dddf724e:36775 {}] hbase.ChoreService(370): Chore service for: master/c302dddf724e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T00:48:20,138 INFO [M:0;c302dddf724e:36775 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T00:48:20,139 DEBUG [M:0;c302dddf724e:36775 {}] master.HMaster(1795): Stopping service threads 2024-11-08T00:48:20,139 INFO [M:0;c302dddf724e:36775 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T00:48:20,139 INFO [M:0;c302dddf724e:36775 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:48:20,139 INFO [M:0;c302dddf724e:36775 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:48:20,139 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T00:48:20,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T00:48:20,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:48:20,140 DEBUG [M:0;c302dddf724e:36775 {}] zookeeper.ZKUtil(347): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T00:48:20,140 WARN [M:0;c302dddf724e:36775 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T00:48:20,140 INFO [M:0;c302dddf724e:36775 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/.lastflushedseqids 2024-11-08T00:48:20,147 INFO [M:0;c302dddf724e:36775 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:48:20,148 INFO [M:0;c302dddf724e:36775 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T00:48:20,148 DEBUG [M:0;c302dddf724e:36775 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T00:48:20,148 INFO [M:0;c302dddf724e:36775 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:48:20,148 DEBUG [M:0;c302dddf724e:36775 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:48:20,148 DEBUG [M:0;c302dddf724e:36775 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T00:48:20,148 DEBUG [M:0;c302dddf724e:36775 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:48:20,148 INFO [M:0;c302dddf724e:36775 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.84 KB heapSize=41.09 KB 2024-11-08T00:48:20,158 DEBUG [M:0;c302dddf724e:36775 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5834622975804119885debd9ee3b930c 2024-11-08T00:48:20,171 DEBUG [M:0;c302dddf724e:36775 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5834622975804119885debd9ee3b930c is 82, key is hbase:meta,,1/info:regioninfo/1731026897017/Put/seqid=0 2024-11-08T00:48:20,171 INFO [M:0;c302dddf724e:36775 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5834622975804119885debd9ee3b930c 2024-11-08T00:48:20,180 DEBUG [M:0;c302dddf724e:36775 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d6d9013002e045ea8a0e10ceb087b111 2024-11-08T00:48:20,193 DEBUG [M:0;c302dddf724e:36775 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d6d9013002e045ea8a0e10ceb087b111 is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731026897466/Put/seqid=0 2024-11-08T00:48:20,194 INFO [M:0;c302dddf724e:36775 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.10 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d6d9013002e045ea8a0e10ceb087b111 2024-11-08T00:48:20,201 DEBUG [M:0;c302dddf724e:36775 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c65b627724a14e6cb64ad78638089b60 2024-11-08T00:48:20,215 DEBUG [M:0;c302dddf724e:36775 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c65b627724a14e6cb64ad78638089b60 is 69, key is c302dddf724e,37111,1731026899305/rs:state/1731026899342/Put/seqid=0 2024-11-08T00:48:20,215 INFO [M:0;c302dddf724e:36775 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c65b627724a14e6cb64ad78638089b60 2024-11-08T00:48:20,217 INFO [M:0;c302dddf724e:36775 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c65b627724a14e6cb64ad78638089b60 2024-11-08T00:48:20,218 DEBUG [M:0;c302dddf724e:36775 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5834622975804119885debd9ee3b930c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5834622975804119885debd9ee3b930c 2024-11-08T00:48:20,220 INFO [M:0;c302dddf724e:36775 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5834622975804119885debd9ee3b930c, entries=8, sequenceid=83, filesize=5.5 K 2024-11-08T00:48:20,221 DEBUG [M:0;c302dddf724e:36775 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d6d9013002e045ea8a0e10ceb087b111 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d6d9013002e045ea8a0e10ceb087b111 2024-11-08T00:48:20,222 INFO [M:0;c302dddf724e:36775 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d6d9013002e045ea8a0e10ceb087b111, entries=9, sequenceid=83, filesize=6.4 K 2024-11-08T00:48:20,223 DEBUG [M:0;c302dddf724e:36775 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c65b627724a14e6cb64ad78638089b60 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c65b627724a14e6cb64ad78638089b60 2024-11-08T00:48:20,224 INFO [M:0;c302dddf724e:36775 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c65b627724a14e6cb64ad78638089b60 2024-11-08T00:48:20,224 INFO [M:0;c302dddf724e:36775 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c916047-a9cc-fcca-cacc-8382e39b81e8/8a8ee971-7e61-25d4-e3e2-329237081e77/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c65b627724a14e6cb64ad78638089b60, entries=3, sequenceid=83, filesize=5.3 K 2024-11-08T00:48:20,225 INFO [M:0;c302dddf724e:36775 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 77ms, sequenceid=83, compaction requested=false 2024-11-08T00:48:20,226 INFO [M:0;c302dddf724e:36775 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:48:20,227 DEBUG [M:0;c302dddf724e:36775 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731026900148Disabling compacts and flushes for region at 1731026900148Disabling writes for close at 1731026900148Obtaining lock to block concurrent updates at 1731026900148Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731026900148Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33626, getHeapSize=42008, getOffHeapSize=0, getCellsCount=98 at 1731026900149 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731026900149Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731026900149Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731026900170 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731026900170Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731026900173 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731026900193 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731026900193Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731026900195 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731026900214 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731026900214Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b00954: reopening flushed file at 1731026900217 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17a2de7e: reopening flushed file at 1731026900220 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cf45928: reopening flushed file at 1731026900222 (+2 ms)Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 77ms, sequenceid=83, compaction requested=false at 1731026900225 (+3 ms)Writing region close event to WAL at 1731026900226 (+1 ms)Closed at 1731026900226 2024-11-08T00:48:20,228 INFO [M:0;c302dddf724e:36775 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T00:48:20,228 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T00:48:20,228 INFO [M:0;c302dddf724e:36775 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36775 2024-11-08T00:48:20,228 INFO [M:0;c302dddf724e:36775 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T00:48:20,236 INFO [RS:0;c302dddf724e:45629 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T00:48:20,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:48:20,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45629-0x100f50c2eb70001, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:48:20,236 INFO [RS:0;c302dddf724e:45629 {}] regionserver.HRegionServer(1031): Exiting; stopping=c302dddf724e,45629,1731026895993; zookeeper connection closed. 2024-11-08T00:48:20,237 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@40968e96 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@40968e96 2024-11-08T00:48:20,237 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-08T00:48:20,330 INFO [M:0;c302dddf724e:36775 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T00:48:20,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:48:20,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36775-0x100f50c2eb70000, quorum=127.0.0.1:58229, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:48:20,333 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T00:48:20,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-08T00:48:20,334 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:48:20,341 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 50) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=770 (was 803), ProcessCount=11 (was 11), AvailableMemoryMB=2867 (was 3122)