2024-11-09 16:48:55,098 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-09 16:48:55,120 main DEBUG Took 0.010053 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-09 16:48:55,121 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-09 16:48:55,121 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-09 16:48:55,123 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-09 16:48:55,125 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 16:48:55,135 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-09 16:48:55,151 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 16:48:55,154 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 16:48:55,154 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 16:48:55,155 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 16:48:55,156 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 16:48:55,156 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 16:48:55,157 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 16:48:55,158 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 16:48:55,158 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 16:48:55,159 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 16:48:55,160 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 16:48:55,161 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 16:48:55,161 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 16:48:55,162 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 16:48:55,162 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 16:48:55,163 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 16:48:55,163 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 16:48:55,164 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 16:48:55,164 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 16:48:55,165 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 16:48:55,166 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 16:48:55,166 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 16:48:55,167 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 16:48:55,167 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 16:48:55,168 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 16:48:55,168 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-09 16:48:55,170 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 16:48:55,172 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-09 16:48:55,174 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-09 16:48:55,175 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-09 16:48:55,177 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-09 16:48:55,177 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-09 16:48:55,189 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-09 16:48:55,192 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-09 16:48:55,194 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-09 16:48:55,195 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-09 16:48:55,195 main DEBUG createAppenders(={Console}) 2024-11-09 16:48:55,196 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-09 16:48:55,196 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-09 16:48:55,197 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-09 16:48:55,197 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-09 16:48:55,198 main DEBUG OutputStream closed 2024-11-09 16:48:55,198 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-09 16:48:55,198 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-09 16:48:55,198 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-09 16:48:55,275 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-09 16:48:55,277 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-09 16:48:55,279 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-09 16:48:55,280 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-09 16:48:55,281 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-09 16:48:55,281 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-09 16:48:55,282 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-09 16:48:55,282 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-09 16:48:55,283 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-09 16:48:55,283 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-09 16:48:55,283 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-09 16:48:55,284 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-09 16:48:55,284 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-09 16:48:55,285 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-09 16:48:55,285 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-09 16:48:55,286 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-09 16:48:55,286 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-09 16:48:55,287 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-09 16:48:55,290 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09 16:48:55,290 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-09 16:48:55,291 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-09 16:48:55,291 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-09T16:48:55,311 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-09 16:48:55,314 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-09 16:48:55,314 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09T16:48:55,601 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c 2024-11-09T16:48:55,628 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=166, ProcessCount=11, AvailableMemoryMB=9080 2024-11-09T16:48:55,726 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-09T16:48:55,824 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:48:55,842 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/cluster_6e3475de-f889-ec92-68f1-42ed4e42dfa9, deleteOnExit=true 2024-11-09T16:48:55,950 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/cluster_6e3475de-f889-ec92-68f1-42ed4e42dfa9/zookeeper_0, clientPort=54118, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/cluster_6e3475de-f889-ec92-68f1-42ed4e42dfa9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/cluster_6e3475de-f889-ec92-68f1-42ed4e42dfa9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-09T16:48:55,962 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54118 2024-11-09T16:48:55,963 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:48:56,063 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18 with version=8 2024-11-09T16:48:56,064 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/hbase-staging 2024-11-09T16:48:56,166 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-09T16:48:56,465 INFO [Time-limited test {}] client.ConnectionUtils(128): master/95f00e0c2391:0 server-side Connection retries=45 2024-11-09T16:48:56,476 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T16:48:56,478 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T16:48:56,489 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T16:48:56,489 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T16:48:56,489 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T16:48:56,644 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-09T16:48:56,750 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T16:48:56,788 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-09T16:48:56,800 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-09T16:48:56,805 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T16:48:56,853 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 12833 (auto-detected) 2024-11-09T16:48:56,855 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-09T16:48:56,884 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43031 2024-11-09T16:48:56,909 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43031 connecting to ZooKeeper ensemble=127.0.0.1:54118 2024-11-09T16:48:56,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:430310x0, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T16:48:56,948 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43031-0x1018def99e00000 connected 2024-11-09T16:48:56,974 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:48:56,975 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:48:56,983 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T16:48:56,987 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18, hbase.cluster.distributed=false 2024-11-09T16:48:57,011 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T16:48:57,016 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43031 2024-11-09T16:48:57,017 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43031 2024-11-09T16:48:57,017 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43031 2024-11-09T16:48:57,018 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43031 2024-11-09T16:48:57,018 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43031 2024-11-09T16:48:57,107 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/95f00e0c2391:0 server-side Connection retries=45 2024-11-09T16:48:57,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T16:48:57,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T16:48:57,109 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T16:48:57,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T16:48:57,110 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T16:48:57,113 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T16:48:57,115 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T16:48:57,116 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35523 2024-11-09T16:48:57,118 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35523 connecting to ZooKeeper ensemble=127.0.0.1:54118 2024-11-09T16:48:57,119 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:48:57,119 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:48:57,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:355230x0, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T16:48:57,125 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35523-0x1018def99e00001 connected 2024-11-09T16:48:57,125 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T16:48:57,131 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T16:48:57,143 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/bucket.cache 2024-11-09T16:48:57,149 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-09T16:48:57,150 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-09T16:48:57,152 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-09T16:48:57,153 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-09T16:48:57,153 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-09T16:48:57,159 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-09T16:48:57,162 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-09T16:48:57,162 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/bucket.persistence is complete. 2024-11-09T16:48:57,166 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T16:48:57,169 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T16:48:57,176 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T16:48:57,183 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35523 2024-11-09T16:48:57,183 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35523 2024-11-09T16:48:57,184 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35523 2024-11-09T16:48:57,198 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35523 2024-11-09T16:48:57,198 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35523 2024-11-09T16:48:57,218 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/95f00e0c2391:0 server-side Connection retries=45 2024-11-09T16:48:57,218 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T16:48:57,218 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T16:48:57,218 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T16:48:57,219 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T16:48:57,219 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T16:48:57,219 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T16:48:57,219 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T16:48:57,220 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35901 2024-11-09T16:48:57,222 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35901 connecting to ZooKeeper ensemble=127.0.0.1:54118 2024-11-09T16:48:57,223 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:48:57,224 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:48:57,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:359010x0, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T16:48:57,228 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35901-0x1018def99e00002 connected 2024-11-09T16:48:57,228 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T16:48:57,229 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T16:48:57,232 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/bucket.cache 2024-11-09T16:48:57,232 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-09T16:48:57,233 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-09T16:48:57,233 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-09T16:48:57,234 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-09T16:48:57,234 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-09T16:48:57,234 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-09T16:48:57,235 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/bucket.persistence is complete. 2024-11-09T16:48:57,235 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-09T16:48:57,236 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T16:48:57,237 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T16:48:57,240 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T16:48:57,244 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35901 2024-11-09T16:48:57,245 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35901 2024-11-09T16:48:57,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35901 2024-11-09T16:48:57,249 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35901 2024-11-09T16:48:57,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35901 2024-11-09T16:48:57,275 DEBUG [M:0;95f00e0c2391:43031 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;95f00e0c2391:43031 2024-11-09T16:48:57,278 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/95f00e0c2391,43031,1731170936251 2024-11-09T16:48:57,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T16:48:57,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T16:48:57,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T16:48:57,292 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/95f00e0c2391,43031,1731170936251 2024-11-09T16:48:57,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T16:48:57,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:48:57,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T16:48:57,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:48:57,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:48:57,330 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T16:48:57,331 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/95f00e0c2391,43031,1731170936251 from backup master directory 2024-11-09T16:48:57,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/95f00e0c2391,43031,1731170936251 2024-11-09T16:48:57,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T16:48:57,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T16:48:57,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T16:48:57,336 WARN [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T16:48:57,336 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=95f00e0c2391,43031,1731170936251 2024-11-09T16:48:57,340 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-09T16:48:57,342 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-09T16:48:57,352 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/hbase.id] with ID: 440966f0-501b-4c04-bd8a-cc36d5a1b6bb 2024-11-09T16:48:57,352 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/.tmp/hbase.id 2024-11-09T16:48:57,359 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/hbase.id] 2024-11-09T16:48:57,402 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:48:57,408 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-09T16:48:57,431 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-09T16:48:57,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:48:57,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:48:57,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:48:57,470 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T16:48:57,474 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-09T16:48:57,527 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T16:48:57,534 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T16:48:57,596 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store 2024-11-09T16:48:57,625 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-09T16:48:57,630 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T16:48:57,632 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T16:48:57,632 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T16:48:57,633 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T16:48:57,635 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T16:48:57,635 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T16:48:57,635 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T16:48:57,636 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731170937632Disabling compacts and flushes for region at 1731170937632Disabling writes for close at 1731170937635 (+3 ms)Writing region close event to WAL at 1731170937635Closed at 1731170937635 2024-11-09T16:48:57,637 WARN [master/95f00e0c2391:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/.initializing 2024-11-09T16:48:57,637 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/WALs/95f00e0c2391,43031,1731170936251 2024-11-09T16:48:57,657 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T16:48:57,673 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95f00e0c2391%2C43031%2C1731170936251, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/WALs/95f00e0c2391,43031,1731170936251, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/oldWALs, maxLogs=10 2024-11-09T16:48:57,712 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/WALs/95f00e0c2391,43031,1731170936251/95f00e0c2391%2C43031%2C1731170936251.1731170937679 2024-11-09T16:48:57,713 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-09T16:48:57,714 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-09T16:48:57,715 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T16:48:57,718 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:48:57,719 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:48:57,769 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:48:57,811 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-09T16:48:57,817 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:48:57,820 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T16:48:57,820 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:48:57,826 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-09T16:48:57,827 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:48:57,827 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T16:48:57,828 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:48:57,833 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-09T16:48:57,834 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:48:57,834 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T16:48:57,835 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:48:57,841 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-09T16:48:57,841 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:48:57,841 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T16:48:57,842 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:48:57,847 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:48:57,848 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:48:57,852 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:48:57,853 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:48:57,856 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T16:48:57,859 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:48:57,870 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T16:48:57,872 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66376336, jitterRate=-0.010915517807006836}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T16:48:57,882 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731170937732Initializing all the Stores at 1731170937732Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731170937733 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731170937734 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731170937734Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731170937734Cleaning up temporary data from old regions at 1731170937853 (+119 ms)Region opened successfully at 1731170937882 (+29 ms) 2024-11-09T16:48:57,884 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-09T16:48:57,920 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36085e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95f00e0c2391/172.17.0.2:0 2024-11-09T16:48:57,956 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-09T16:48:57,974 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-09T16:48:57,974 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-09T16:48:57,979 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T16:48:57,980 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T16:48:57,988 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-09T16:48:57,989 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-09T16:48:58,024 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-09T16:48:58,036 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-09T16:48:58,039 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-09T16:48:58,042 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-09T16:48:58,044 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-09T16:48:58,046 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-09T16:48:58,049 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-09T16:48:58,054 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-09T16:48:58,056 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-09T16:48:58,058 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-09T16:48:58,059 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-09T16:48:58,079 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-09T16:48:58,081 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-09T16:48:58,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T16:48:58,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T16:48:58,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T16:48:58,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:48:58,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:48:58,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:48:58,103 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=95f00e0c2391,43031,1731170936251, sessionid=0x1018def99e00000, setting cluster-up flag (Was=false) 2024-11-09T16:48:58,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:48:58,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:48:58,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:48:58,125 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-09T16:48:58,127 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=95f00e0c2391,43031,1731170936251 2024-11-09T16:48:58,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:48:58,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:48:58,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:48:58,139 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-09T16:48:58,142 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=95f00e0c2391,43031,1731170936251 2024-11-09T16:48:58,148 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-09T16:48:58,155 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.HRegionServer(746): ClusterId : 440966f0-501b-4c04-bd8a-cc36d5a1b6bb 2024-11-09T16:48:58,155 INFO [RS:1;95f00e0c2391:35901 {}] regionserver.HRegionServer(746): ClusterId : 440966f0-501b-4c04-bd8a-cc36d5a1b6bb 2024-11-09T16:48:58,158 DEBUG [RS:1;95f00e0c2391:35901 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T16:48:58,158 DEBUG [RS:0;95f00e0c2391:35523 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T16:48:58,165 DEBUG [RS:0;95f00e0c2391:35523 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T16:48:58,165 DEBUG [RS:1;95f00e0c2391:35901 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T16:48:58,165 DEBUG [RS:0;95f00e0c2391:35523 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T16:48:58,165 DEBUG [RS:1;95f00e0c2391:35901 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T16:48:58,169 DEBUG [RS:1;95f00e0c2391:35901 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T16:48:58,169 DEBUG [RS:0;95f00e0c2391:35523 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T16:48:58,170 DEBUG [RS:1;95f00e0c2391:35901 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e8403f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95f00e0c2391/172.17.0.2:0 2024-11-09T16:48:58,170 DEBUG [RS:0;95f00e0c2391:35523 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ea92cbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95f00e0c2391/172.17.0.2:0 2024-11-09T16:48:58,187 DEBUG [RS:0;95f00e0c2391:35523 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;95f00e0c2391:35523 2024-11-09T16:48:58,187 DEBUG [RS:1;95f00e0c2391:35901 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;95f00e0c2391:35901 2024-11-09T16:48:58,190 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T16:48:58,190 INFO [RS:1;95f00e0c2391:35901 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T16:48:58,190 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T16:48:58,190 INFO [RS:1;95f00e0c2391:35901 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T16:48:58,190 DEBUG [RS:0;95f00e0c2391:35523 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T16:48:58,190 DEBUG [RS:1;95f00e0c2391:35901 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T16:48:58,194 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.HRegionServer(2659): reportForDuty to master=95f00e0c2391,43031,1731170936251 with port=35523, startcode=1731170937067 2024-11-09T16:48:58,194 INFO [RS:1;95f00e0c2391:35901 {}] regionserver.HRegionServer(2659): reportForDuty to master=95f00e0c2391,43031,1731170936251 with port=35901, startcode=1731170937217 2024-11-09T16:48:58,210 DEBUG [RS:0;95f00e0c2391:35523 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T16:48:58,210 DEBUG [RS:1;95f00e0c2391:35901 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T16:48:58,241 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T16:48:58,254 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-09T16:48:58,263 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-09T16:48:58,271 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 95f00e0c2391,43031,1731170936251 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-09T16:48:58,281 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/95f00e0c2391:0, corePoolSize=5, maxPoolSize=5 2024-11-09T16:48:58,282 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/95f00e0c2391:0, corePoolSize=5, maxPoolSize=5 2024-11-09T16:48:58,282 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/95f00e0c2391:0, corePoolSize=5, maxPoolSize=5 2024-11-09T16:48:58,282 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/95f00e0c2391:0, corePoolSize=5, maxPoolSize=5 2024-11-09T16:48:58,283 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/95f00e0c2391:0, corePoolSize=10, maxPoolSize=10 2024-11-09T16:48:58,283 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,283 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/95f00e0c2391:0, corePoolSize=2, maxPoolSize=2 2024-11-09T16:48:58,284 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,300 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731170968300 2024-11-09T16:48:58,301 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58875, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T16:48:58,301 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35973, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T16:48:58,303 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-09T16:48:58,304 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T16:48:58,304 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-09T16:48:58,304 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-09T16:48:58,307 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-09T16:48:58,307 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-09T16:48:58,308 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-09T16:48:58,308 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-09T16:48:58,312 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43031 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T16:48:58,314 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:48:58,312 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,315 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-09T16:48:58,316 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-09T16:48:58,318 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-09T16:48:58,318 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-09T16:48:58,320 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43031 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T16:48:58,321 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-09T16:48:58,321 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-09T16:48:58,323 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/95f00e0c2391:0:becomeActiveMaster-HFileCleaner.large.0-1731170938323,5,FailOnTimeoutGroup] 2024-11-09T16:48:58,325 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/95f00e0c2391:0:becomeActiveMaster-HFileCleaner.small.0-1731170938323,5,FailOnTimeoutGroup] 2024-11-09T16:48:58,325 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,326 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-09T16:48:58,327 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,327 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,344 DEBUG [RS:0;95f00e0c2391:35523 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-09T16:48:58,344 DEBUG [RS:1;95f00e0c2391:35901 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-09T16:48:58,345 WARN [RS:0;95f00e0c2391:35523 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-09T16:48:58,345 WARN [RS:1;95f00e0c2391:35901 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-09T16:48:58,345 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-09T16:48:58,345 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18 2024-11-09T16:48:58,353 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T16:48:58,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-09T16:48:58,365 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T16:48:58,365 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:48:58,365 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T16:48:58,366 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-09T16:48:58,370 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T16:48:58,370 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:48:58,370 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T16:48:58,370 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-09T16:48:58,374 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T16:48:58,374 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:48:58,374 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T16:48:58,375 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-09T16:48:58,378 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T16:48:58,379 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:48:58,379 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T16:48:58,379 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T16:48:58,380 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740 2024-11-09T16:48:58,380 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740 2024-11-09T16:48:58,381 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T16:48:58,381 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T16:48:58,381 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T16:48:58,382 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T16:48:58,389 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T16:48:58,389 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64895317, jitterRate=-0.0329844206571579}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T16:48:58,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731170938354Initializing all the Stores at 1731170938354Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731170938354Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731170938354Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731170938354Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731170938354Cleaning up temporary data from old regions at 1731170938381 (+27 ms)Region opened successfully at 1731170938392 (+11 ms) 2024-11-09T16:48:58,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T16:48:58,392 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T16:48:58,393 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T16:48:58,393 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T16:48:58,393 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T16:48:58,395 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T16:48:58,395 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731170938392Disabling compacts and flushes for region at 1731170938392Disabling writes for close at 1731170938393 (+1 ms)Writing region close event to WAL at 1731170938394 (+1 ms)Closed at 1731170938395 (+1 ms) 2024-11-09T16:48:58,398 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T16:48:58,398 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-09T16:48:58,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T16:48:58,415 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T16:48:58,417 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-09T16:48:58,446 INFO [RS:1;95f00e0c2391:35901 {}] regionserver.HRegionServer(2659): reportForDuty to master=95f00e0c2391,43031,1731170936251 with port=35901, startcode=1731170937217 2024-11-09T16:48:58,446 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.HRegionServer(2659): reportForDuty to master=95f00e0c2391,43031,1731170936251 with port=35523, startcode=1731170937067 2024-11-09T16:48:58,449 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43031 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 95f00e0c2391,35901,1731170937217 2024-11-09T16:48:58,453 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43031 {}] master.ServerManager(517): Registering regionserver=95f00e0c2391,35901,1731170937217 2024-11-09T16:48:58,461 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43031 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 95f00e0c2391,35523,1731170937067 2024-11-09T16:48:58,461 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43031 {}] master.ServerManager(517): Registering regionserver=95f00e0c2391,35523,1731170937067 2024-11-09T16:48:58,462 DEBUG [RS:1;95f00e0c2391:35901 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18 2024-11-09T16:48:58,462 DEBUG [RS:1;95f00e0c2391:35901 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-09T16:48:58,462 DEBUG [RS:1;95f00e0c2391:35901 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T16:48:58,464 DEBUG [RS:0;95f00e0c2391:35523 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18 2024-11-09T16:48:58,464 DEBUG [RS:0;95f00e0c2391:35523 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-09T16:48:58,464 DEBUG [RS:0;95f00e0c2391:35523 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T16:48:58,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T16:48:58,472 DEBUG [RS:0;95f00e0c2391:35523 {}] zookeeper.ZKUtil(111): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/95f00e0c2391,35523,1731170937067 2024-11-09T16:48:58,473 WARN [RS:0;95f00e0c2391:35523 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T16:48:58,473 INFO [RS:0;95f00e0c2391:35523 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T16:48:58,473 DEBUG [RS:0;95f00e0c2391:35523 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,35523,1731170937067 2024-11-09T16:48:58,474 DEBUG [RS:1;95f00e0c2391:35901 {}] zookeeper.ZKUtil(111): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/95f00e0c2391,35901,1731170937217 2024-11-09T16:48:58,474 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [95f00e0c2391,35901,1731170937217] 2024-11-09T16:48:58,474 WARN [RS:1;95f00e0c2391:35901 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T16:48:58,474 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [95f00e0c2391,35523,1731170937067] 2024-11-09T16:48:58,474 INFO [RS:1;95f00e0c2391:35901 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T16:48:58,474 DEBUG [RS:1;95f00e0c2391:35901 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,35901,1731170937217 2024-11-09T16:48:58,506 INFO [RS:1;95f00e0c2391:35901 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T16:48:58,506 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T16:48:58,526 INFO [RS:1;95f00e0c2391:35901 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T16:48:58,526 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T16:48:58,531 INFO [RS:1;95f00e0c2391:35901 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T16:48:58,532 INFO [RS:1;95f00e0c2391:35901 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,532 INFO [RS:0;95f00e0c2391:35523 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T16:48:58,532 INFO [RS:0;95f00e0c2391:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,533 INFO [RS:1;95f00e0c2391:35901 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T16:48:58,533 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T16:48:58,540 INFO [RS:1;95f00e0c2391:35901 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T16:48:58,540 INFO [RS:0;95f00e0c2391:35523 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T16:48:58,542 INFO [RS:1;95f00e0c2391:35901 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,542 INFO [RS:0;95f00e0c2391:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,542 DEBUG [RS:1;95f00e0c2391:35901 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,542 DEBUG [RS:0;95f00e0c2391:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,543 DEBUG [RS:1;95f00e0c2391:35901 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,543 DEBUG [RS:0;95f00e0c2391:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,543 DEBUG [RS:1;95f00e0c2391:35901 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,543 DEBUG [RS:0;95f00e0c2391:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,543 DEBUG [RS:1;95f00e0c2391:35901 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,543 DEBUG [RS:0;95f00e0c2391:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,543 DEBUG [RS:1;95f00e0c2391:35901 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,544 DEBUG [RS:0;95f00e0c2391:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,544 DEBUG [RS:1;95f00e0c2391:35901 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/95f00e0c2391:0, corePoolSize=2, maxPoolSize=2 2024-11-09T16:48:58,544 DEBUG [RS:0;95f00e0c2391:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/95f00e0c2391:0, corePoolSize=2, maxPoolSize=2 2024-11-09T16:48:58,544 DEBUG [RS:1;95f00e0c2391:35901 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,544 DEBUG [RS:0;95f00e0c2391:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,544 DEBUG [RS:1;95f00e0c2391:35901 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,544 DEBUG [RS:0;95f00e0c2391:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,544 DEBUG [RS:1;95f00e0c2391:35901 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,545 DEBUG [RS:0;95f00e0c2391:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,545 DEBUG [RS:1;95f00e0c2391:35901 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,545 DEBUG [RS:0;95f00e0c2391:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,545 DEBUG [RS:1;95f00e0c2391:35901 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,545 DEBUG [RS:0;95f00e0c2391:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,545 DEBUG [RS:1;95f00e0c2391:35901 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,545 DEBUG [RS:0;95f00e0c2391:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:48:58,545 DEBUG [RS:1;95f00e0c2391:35901 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/95f00e0c2391:0, corePoolSize=3, maxPoolSize=3 2024-11-09T16:48:58,546 DEBUG [RS:0;95f00e0c2391:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/95f00e0c2391:0, corePoolSize=3, maxPoolSize=3 2024-11-09T16:48:58,546 DEBUG [RS:1;95f00e0c2391:35901 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/95f00e0c2391:0, corePoolSize=3, maxPoolSize=3 2024-11-09T16:48:58,546 DEBUG [RS:0;95f00e0c2391:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/95f00e0c2391:0, corePoolSize=3, maxPoolSize=3 2024-11-09T16:48:58,547 INFO [RS:1;95f00e0c2391:35901 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,547 INFO [RS:1;95f00e0c2391:35901 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,547 INFO [RS:1;95f00e0c2391:35901 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,547 INFO [RS:1;95f00e0c2391:35901 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,548 INFO [RS:1;95f00e0c2391:35901 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,548 INFO [RS:1;95f00e0c2391:35901 {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,35901,1731170937217-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T16:48:58,550 INFO [RS:0;95f00e0c2391:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,550 INFO [RS:0;95f00e0c2391:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,550 INFO [RS:0;95f00e0c2391:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,550 INFO [RS:0;95f00e0c2391:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,551 INFO [RS:0;95f00e0c2391:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,551 INFO [RS:0;95f00e0c2391:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,35523,1731170937067-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T16:48:58,569 WARN [95f00e0c2391:43031 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T16:48:58,585 INFO [RS:1;95f00e0c2391:35901 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T16:48:58,585 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T16:48:58,588 INFO [RS:1;95f00e0c2391:35901 {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,35901,1731170937217-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,588 INFO [RS:0;95f00e0c2391:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,35523,1731170937067-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,588 INFO [RS:1;95f00e0c2391:35901 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,588 INFO [RS:0;95f00e0c2391:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,589 INFO [RS:1;95f00e0c2391:35901 {}] regionserver.Replication(171): 95f00e0c2391,35901,1731170937217 started 2024-11-09T16:48:58,589 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.Replication(171): 95f00e0c2391,35523,1731170937067 started 2024-11-09T16:48:58,615 INFO [RS:1;95f00e0c2391:35901 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,616 INFO [RS:1;95f00e0c2391:35901 {}] regionserver.HRegionServer(1482): Serving as 95f00e0c2391,35901,1731170937217, RpcServer on 95f00e0c2391/172.17.0.2:35901, sessionid=0x1018def99e00002 2024-11-09T16:48:58,616 INFO [RS:0;95f00e0c2391:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:58,616 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.HRegionServer(1482): Serving as 95f00e0c2391,35523,1731170937067, RpcServer on 95f00e0c2391/172.17.0.2:35523, sessionid=0x1018def99e00001 2024-11-09T16:48:58,617 DEBUG [RS:1;95f00e0c2391:35901 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T16:48:58,617 DEBUG [RS:0;95f00e0c2391:35523 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T16:48:58,617 DEBUG [RS:1;95f00e0c2391:35901 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 95f00e0c2391,35901,1731170937217 2024-11-09T16:48:58,617 DEBUG [RS:0;95f00e0c2391:35523 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 95f00e0c2391,35523,1731170937067 2024-11-09T16:48:58,618 DEBUG [RS:1;95f00e0c2391:35901 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95f00e0c2391,35901,1731170937217' 2024-11-09T16:48:58,618 DEBUG [RS:0;95f00e0c2391:35523 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95f00e0c2391,35523,1731170937067' 2024-11-09T16:48:58,618 DEBUG [RS:1;95f00e0c2391:35901 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T16:48:58,618 DEBUG [RS:0;95f00e0c2391:35523 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T16:48:58,619 DEBUG [RS:1;95f00e0c2391:35901 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T16:48:58,619 DEBUG [RS:0;95f00e0c2391:35523 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T16:48:58,620 DEBUG [RS:1;95f00e0c2391:35901 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T16:48:58,620 DEBUG [RS:0;95f00e0c2391:35523 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T16:48:58,620 DEBUG [RS:1;95f00e0c2391:35901 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T16:48:58,621 DEBUG [RS:0;95f00e0c2391:35523 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T16:48:58,621 DEBUG [RS:1;95f00e0c2391:35901 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 95f00e0c2391,35901,1731170937217 2024-11-09T16:48:58,621 DEBUG [RS:1;95f00e0c2391:35901 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95f00e0c2391,35901,1731170937217' 2024-11-09T16:48:58,621 DEBUG [RS:1;95f00e0c2391:35901 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T16:48:58,621 DEBUG [RS:0;95f00e0c2391:35523 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 95f00e0c2391,35523,1731170937067 2024-11-09T16:48:58,621 DEBUG [RS:0;95f00e0c2391:35523 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95f00e0c2391,35523,1731170937067' 2024-11-09T16:48:58,621 DEBUG [RS:0;95f00e0c2391:35523 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T16:48:58,622 DEBUG [RS:1;95f00e0c2391:35901 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T16:48:58,622 DEBUG [RS:0;95f00e0c2391:35523 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T16:48:58,623 DEBUG [RS:1;95f00e0c2391:35901 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T16:48:58,623 DEBUG [RS:0;95f00e0c2391:35523 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T16:48:58,623 INFO [RS:1;95f00e0c2391:35901 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T16:48:58,623 INFO [RS:0;95f00e0c2391:35523 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T16:48:58,623 INFO [RS:1;95f00e0c2391:35901 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T16:48:58,623 INFO [RS:0;95f00e0c2391:35523 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T16:48:58,729 INFO [RS:1;95f00e0c2391:35901 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T16:48:58,729 INFO [RS:0;95f00e0c2391:35523 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T16:48:58,730 INFO [RS:1;95f00e0c2391:35901 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=95f00e0c2391%2C35901%2C1731170937217, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,35901,1731170937217, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/oldWALs, maxLogs=55 2024-11-09T16:48:58,730 INFO [RS:0;95f00e0c2391:35523 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=95f00e0c2391%2C35523%2C1731170937067, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,35523,1731170937067, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/oldWALs, maxLogs=55 2024-11-09T16:48:58,755 INFO [RS:1;95f00e0c2391:35901 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,35901,1731170937217/95f00e0c2391%2C35901%2C1731170937217.1731170938735 2024-11-09T16:48:58,755 INFO [RS:0;95f00e0c2391:35523 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,35523,1731170937067/95f00e0c2391%2C35523%2C1731170937067.1731170938735 2024-11-09T16:48:58,756 DEBUG [RS:0;95f00e0c2391:35523 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-09T16:48:58,756 DEBUG [RS:1;95f00e0c2391:35901 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-09T16:48:58,821 DEBUG [95f00e0c2391:43031 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-09T16:48:58,833 DEBUG [95f00e0c2391:43031 {}] balancer.BalancerClusterState(204): Hosts are {95f00e0c2391=0} racks are {/default-rack=0} 2024-11-09T16:48:58,837 DEBUG [95f00e0c2391:43031 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-09T16:48:58,837 DEBUG [95f00e0c2391:43031 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-09T16:48:58,837 DEBUG [95f00e0c2391:43031 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-09T16:48:58,837 DEBUG [95f00e0c2391:43031 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-09T16:48:58,837 INFO [95f00e0c2391:43031 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-09T16:48:58,837 INFO [95f00e0c2391:43031 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-09T16:48:58,837 DEBUG [95f00e0c2391:43031 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-09T16:48:58,845 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=95f00e0c2391,35523,1731170937067 2024-11-09T16:48:58,851 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 95f00e0c2391,35523,1731170937067, state=OPENING 2024-11-09T16:48:58,856 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-09T16:48:58,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:48:58,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:48:58,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:48:58,859 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T16:48:58,859 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T16:48:58,860 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T16:48:58,861 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T16:48:58,863 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=95f00e0c2391,35523,1731170937067}] 2024-11-09T16:48:59,036 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T16:48:59,039 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37555, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T16:48:59,048 INFO [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-09T16:48:59,049 INFO [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T16:48:59,049 INFO [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-09T16:48:59,050 INFO [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=95f00e0c2391%2C35523%2C1731170937067.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,35523,1731170937067, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/oldWALs, maxLogs=55 2024-11-09T16:48:59,071 INFO [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,35523,1731170937067/95f00e0c2391%2C35523%2C1731170937067.meta.1731170939052.meta 2024-11-09T16:48:59,072 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-09T16:48:59,072 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-09T16:48:59,074 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-09T16:48:59,076 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-09T16:48:59,081 INFO [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-09T16:48:59,086 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-09T16:48:59,086 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T16:48:59,087 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-09T16:48:59,087 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-09T16:48:59,088 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-09T16:48:59,090 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T16:48:59,090 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:48:59,090 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T16:48:59,090 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-09T16:48:59,091 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T16:48:59,091 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:48:59,091 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T16:48:59,092 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-09T16:48:59,092 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T16:48:59,093 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:48:59,093 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T16:48:59,093 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-09T16:48:59,094 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T16:48:59,094 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:48:59,094 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T16:48:59,094 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T16:48:59,095 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740 2024-11-09T16:48:59,096 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740 2024-11-09T16:48:59,097 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T16:48:59,097 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T16:48:59,097 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T16:48:59,098 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T16:48:59,099 INFO [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65359000, jitterRate=-0.026075005531311035}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T16:48:59,099 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-09T16:48:59,101 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731170939087Writing region info on filesystem at 1731170939088 (+1 ms)Initializing all the Stores at 1731170939088Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731170939088Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731170939088Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731170939088Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731170939088Cleaning up temporary data from old regions at 1731170939097 (+9 ms)Running coprocessor post-open hooks at 1731170939100 (+3 ms)Region opened successfully at 1731170939101 (+1 ms) 2024-11-09T16:48:59,107 INFO [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731170939027 2024-11-09T16:48:59,117 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-09T16:48:59,118 INFO [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-09T16:48:59,119 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=95f00e0c2391,35523,1731170937067 2024-11-09T16:48:59,120 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 95f00e0c2391,35523,1731170937067, state=OPEN 2024-11-09T16:48:59,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T16:48:59,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T16:48:59,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T16:48:59,123 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T16:48:59,123 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T16:48:59,123 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T16:48:59,123 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=95f00e0c2391,35523,1731170937067 2024-11-09T16:48:59,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-09T16:48:59,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=95f00e0c2391,35523,1731170937067 in 261 msec 2024-11-09T16:48:59,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-09T16:48:59,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 723 msec 2024-11-09T16:48:59,132 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T16:48:59,132 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-09T16:48:59,155 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T16:48:59,156 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=95f00e0c2391,35523,1731170937067, seqNum=-1] 2024-11-09T16:48:59,176 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T16:48:59,179 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50433, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T16:48:59,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0180 sec 2024-11-09T16:48:59,202 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731170939202, completionTime=-1 2024-11-09T16:48:59,206 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-09T16:48:59,206 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T16:48:59,231 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T16:48:59,231 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731170999231 2024-11-09T16:48:59,231 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731171059231 2024-11-09T16:48:59,231 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-09T16:48:59,233 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-09T16:48:59,241 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,43031,1731170936251-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:59,241 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,43031,1731170936251-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:59,242 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,43031,1731170936251-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:59,243 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-95f00e0c2391:43031, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:59,243 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:59,244 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:59,249 DEBUG [master/95f00e0c2391:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-09T16:48:59,270 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.932sec 2024-11-09T16:48:59,271 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-09T16:48:59,272 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-09T16:48:59,273 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-09T16:48:59,274 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-09T16:48:59,274 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-09T16:48:59,275 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,43031,1731170936251-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T16:48:59,275 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,43031,1731170936251-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-09T16:48:59,276 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-09T16:48:59,277 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-09T16:48:59,278 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,43031,1731170936251-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T16:48:59,365 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c7dcd51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T16:48:59,367 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-09T16:48:59,367 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-09T16:48:59,371 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 95f00e0c2391,43031,-1 for getting cluster id 2024-11-09T16:48:59,374 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T16:48:59,382 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '440966f0-501b-4c04-bd8a-cc36d5a1b6bb' 2024-11-09T16:48:59,385 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T16:48:59,385 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "440966f0-501b-4c04-bd8a-cc36d5a1b6bb" 2024-11-09T16:48:59,386 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c1fc471, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T16:48:59,386 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [95f00e0c2391,43031,-1] 2024-11-09T16:48:59,388 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T16:48:59,390 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T16:48:59,392 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T16:48:59,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b7a9956, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T16:48:59,396 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T16:48:59,404 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=95f00e0c2391,35523,1731170937067, seqNum=-1] 2024-11-09T16:48:59,405 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T16:48:59,408 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T16:48:59,447 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=95f00e0c2391,43031,1731170936251 2024-11-09T16:48:59,453 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T16:48:59,458 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 95f00e0c2391,43031,1731170936251 2024-11-09T16:48:59,461 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2f6029d2 2024-11-09T16:48:59,462 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T16:48:59,465 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T16:48:59,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731170939448', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T16:48:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731170939448 2024-11-09T16:48:59,479 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731170939448 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T16:48:59,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731170939448" procId is: 4 2024-11-09T16:48:59,481 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:48:59,483 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731170939448 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T16:48:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T16:48:59,496 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731170939448-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 696e42658047076b8f9ad1d9849106e2, NAME => 'testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731170939448', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18 2024-11-09T16:48:59,503 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731170939448-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T16:48:59,503 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731170939448-pool-0 {}] regionserver.HRegion(1722): Closing 696e42658047076b8f9ad1d9849106e2, disabling compactions & flushes 2024-11-09T16:48:59,503 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731170939448-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:48:59,503 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731170939448-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:48:59,504 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731170939448-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. after waiting 1 ms 2024-11-09T16:48:59,504 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731170939448-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:48:59,504 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731170939448-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:48:59,504 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731170939448-pool-0 {}] regionserver.HRegion(1676): Region close journal for 696e42658047076b8f9ad1d9849106e2: Waiting for close lock at 1731170939503Disabling compacts and flushes for region at 1731170939503Disabling writes for close at 1731170939504 (+1 ms)Writing region close event to WAL at 1731170939504Closed at 1731170939504 2024-11-09T16:48:59,505 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731170939448 execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T16:48:59,510 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731170939506"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731170939506"}]},"ts":"1731170939506"} 2024-11-09T16:48:59,514 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T16:48:59,516 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731170939448 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T16:48:59,518 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731170939448","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731170939516"}]},"ts":"1731170939516"} 2024-11-09T16:48:59,522 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731170939448, state=ENABLING in hbase:meta 2024-11-09T16:48:59,523 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {95f00e0c2391=0} racks are {/default-rack=0} 2024-11-09T16:48:59,523 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-09T16:48:59,523 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-09T16:48:59,523 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-09T16:48:59,523 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-09T16:48:59,523 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-09T16:48:59,523 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-09T16:48:59,523 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-09T16:48:59,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731170939448, region=696e42658047076b8f9ad1d9849106e2, ASSIGN}] 2024-11-09T16:48:59,526 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731170939448, region=696e42658047076b8f9ad1d9849106e2, ASSIGN 2024-11-09T16:48:59,527 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731170939448, region=696e42658047076b8f9ad1d9849106e2, ASSIGN; state=OFFLINE, location=95f00e0c2391,35901,1731170937217; forceNewPlan=false, retain=false 2024-11-09T16:48:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T16:48:59,679 INFO [95f00e0c2391:43031 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-09T16:48:59,680 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=696e42658047076b8f9ad1d9849106e2, regionState=OPENING, regionLocation=95f00e0c2391,35901,1731170937217 2024-11-09T16:48:59,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731170939448, region=696e42658047076b8f9ad1d9849106e2, ASSIGN because future has completed 2024-11-09T16:48:59,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 696e42658047076b8f9ad1d9849106e2, server=95f00e0c2391,35901,1731170937217}] 2024-11-09T16:48:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T16:48:59,838 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T16:48:59,840 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47311, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T16:48:59,843 INFO [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:48:59,843 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 696e42658047076b8f9ad1d9849106e2, NAME => 'testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2.', STARTKEY => '', ENDKEY => ''} 2024-11-09T16:48:59,844 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731170939448 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:48:59,844 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T16:48:59,844 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:48:59,844 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:48:59,845 INFO [StoreOpener-696e42658047076b8f9ad1d9849106e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:48:59,850 INFO [StoreOpener-696e42658047076b8f9ad1d9849106e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 696e42658047076b8f9ad1d9849106e2 columnFamilyName family 2024-11-09T16:48:59,850 DEBUG [StoreOpener-696e42658047076b8f9ad1d9849106e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:48:59,850 INFO [StoreOpener-696e42658047076b8f9ad1d9849106e2-1 {}] regionserver.HStore(327): Store=696e42658047076b8f9ad1d9849106e2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T16:48:59,851 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:48:59,852 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2 2024-11-09T16:48:59,852 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2 2024-11-09T16:48:59,852 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:48:59,852 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:48:59,853 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:48:59,860 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T16:48:59,860 INFO [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 696e42658047076b8f9ad1d9849106e2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73570110, jitterRate=0.0962800681591034}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T16:48:59,860 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:48:59,861 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 696e42658047076b8f9ad1d9849106e2: Running coprocessor pre-open hook at 1731170939844Writing region info on filesystem at 1731170939844Initializing all the Stores at 1731170939844Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731170939845 (+1 ms)Cleaning up temporary data from old regions at 1731170939852 (+7 ms)Running coprocessor post-open hooks at 1731170939860 (+8 ms)Region opened successfully at 1731170939861 (+1 ms) 2024-11-09T16:48:59,862 INFO [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2., pid=6, masterSystemTime=1731170939838 2024-11-09T16:48:59,865 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:48:59,866 INFO [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:48:59,866 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=696e42658047076b8f9ad1d9849106e2, regionState=OPEN, openSeqNum=2, regionLocation=95f00e0c2391,35901,1731170937217 2024-11-09T16:48:59,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 696e42658047076b8f9ad1d9849106e2, server=95f00e0c2391,35901,1731170937217 because future has completed 2024-11-09T16:48:59,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-09T16:48:59,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 696e42658047076b8f9ad1d9849106e2, server=95f00e0c2391,35901,1731170937217 in 186 msec 2024-11-09T16:48:59,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-09T16:48:59,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731170939448, region=696e42658047076b8f9ad1d9849106e2, ASSIGN in 349 msec 2024-11-09T16:48:59,876 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731170939448 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T16:48:59,877 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731170939448","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731170939876"}]},"ts":"1731170939876"} 2024-11-09T16:48:59,880 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731170939448, state=ENABLED in hbase:meta 2024-11-09T16:48:59,881 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731170939448 execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T16:48:59,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731170939448 in 408 msec 2024-11-09T16:49:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T16:49:00,121 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731170939448 get assigned. Timeout = 60000ms 2024-11-09T16:49:00,121 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731170939448 completed 2024-11-09T16:49:00,124 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T16:49:00,130 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731170939448 assigned to meta. Checking AM states. 2024-11-09T16:49:00,130 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T16:49:00,131 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731170939448 assigned. 2024-11-09T16:49:00,139 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731170939448', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2., hostname=95f00e0c2391,35901,1731170937217, seqNum=2] 2024-11-09T16:49:00,140 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T16:49:00,143 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T16:49:00,152 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 696e42658047076b8f9ad1d9849106e2 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-09T16:49:00,177 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 17c8e2c7523346868abe2d860a571a60 2024-11-09T16:49:00,216 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2/.tmp/family/17c8e2c7523346868abe2d860a571a60 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-09T16:49:00,222 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2/.tmp/family/17c8e2c7523346868abe2d860a571a60 2024-11-09T16:49:00,260 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2/.tmp/family/17c8e2c7523346868abe2d860a571a60 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2/family/17c8e2c7523346868abe2d860a571a60 2024-11-09T16:49:00,261 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2/family/17c8e2c7523346868abe2d860a571a60, delay=934 ms 2024-11-09T16:49:00,263 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2/family/17c8e2c7523346868abe2d860a571a60, entries=2, sequenceid=6, filesize=4.9 K 2024-11-09T16:49:00,269 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 696e42658047076b8f9ad1d9849106e2 in 116ms, sequenceid=6, compaction requested=false 2024-11-09T16:49:00,271 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731170939448' 2024-11-09T16:49:00,273 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 696e42658047076b8f9ad1d9849106e2: 2024-11-09T16:49:01,198 DEBUG [hfile-prefetch-1731170940261 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 17c8e2c7523346868abe2d860a571a60_0. 2024-11-09T16:49:01,202 DEBUG [hfile-prefetch-1731170940261 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 17c8e2c7523346868abe2d860a571a60_117. 2024-11-09T16:49:01,203 DEBUG [hfile-prefetch-1731170940261 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2/family/17c8e2c7523346868abe2d860a571a60, with total blocks 2, and data blocks 1 2024-11-09T16:49:01,203 DEBUG [hfile-prefetch-1731170940261 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-09T16:49:01,203 DEBUG [hfile-prefetch-1731170940261 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2/family/17c8e2c7523346868abe2d860a571a60, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-09T16:49:01,204 DEBUG [hfile-prefetch-1731170940261 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2/family/17c8e2c7523346868abe2d860a571a60. Will sleep 100ms and try the verification again. 2024-11-09T16:49:01,236 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-09T16:49:01,237 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-09T16:49:01,237 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-09T16:49:01,237 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-09T16:49:01,255 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-09T16:49:01,255 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-09T16:49:01,255 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-09T16:49:01,255 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-09T16:49:01,258 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-09T16:49:01,258 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731170941237 2024-11-09T16:49:01,258 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-09T16:49:01,258 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-09T16:49:01,259 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731170941255 2024-11-09T16:49:01,259 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-09T16:49:01,282 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=696e42658047076b8f9ad1d9849106e2, source=95f00e0c2391,35901,1731170937217, destination=95f00e0c2391,35523,1731170937067, warming up region on 95f00e0c2391,35523,1731170937067 2024-11-09T16:49:01,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=696e42658047076b8f9ad1d9849106e2, source=95f00e0c2391,35901,1731170937217, destination=95f00e0c2391,35523,1731170937067, running balancer 2024-11-09T16:49:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731170939448, region=696e42658047076b8f9ad1d9849106e2, REOPEN/MOVE 2024-11-09T16:49:01,286 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:49:01,286 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731170939448, region=696e42658047076b8f9ad1d9849106e2, REOPEN/MOVE 2024-11-09T16:49:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] regionserver.HRegion(7855): Warmup {ENCODED => 696e42658047076b8f9ad1d9849106e2, NAME => 'testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2.', STARTKEY => '', ENDKEY => ''} 2024-11-09T16:49:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T16:49:01,287 INFO [StoreOpener-696e42658047076b8f9ad1d9849106e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:49:01,288 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=696e42658047076b8f9ad1d9849106e2, regionState=CLOSING, regionLocation=95f00e0c2391,35901,1731170937217 2024-11-09T16:49:01,288 INFO [StoreOpener-696e42658047076b8f9ad1d9849106e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 696e42658047076b8f9ad1d9849106e2 columnFamilyName family 2024-11-09T16:49:01,289 DEBUG [StoreOpener-696e42658047076b8f9ad1d9849106e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:49:01,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731170939448, region=696e42658047076b8f9ad1d9849106e2, REOPEN/MOVE because future has completed 2024-11-09T16:49:01,292 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-09T16:49:01,292 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 696e42658047076b8f9ad1d9849106e2, server=95f00e0c2391,35901,1731170937217}] 2024-11-09T16:49:01,292 DEBUG [StoreOpener-696e42658047076b8f9ad1d9849106e2-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2/family/17c8e2c7523346868abe2d860a571a60 2024-11-09T16:49:01,292 INFO [StoreOpener-696e42658047076b8f9ad1d9849106e2-1 {}] regionserver.HStore(327): Store=696e42658047076b8f9ad1d9849106e2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T16:49:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] regionserver.HRegion(1722): Closing 696e42658047076b8f9ad1d9849106e2, disabling compactions & flushes 2024-11-09T16:49:01,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:49:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:49:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. after waiting 0 ms 2024-11-09T16:49:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:49:01,294 DEBUG [StoreFileCloser-696e42658047076b8f9ad1d9849106e2-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2/family/17c8e2c7523346868abe2d860a571a60 2024-11-09T16:49:01,294 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:49:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] regionserver.HRegion(1676): Region close journal for 696e42658047076b8f9ad1d9849106e2: Waiting for close lock at 1731170941293Disabling compacts and flushes for region at 1731170941293Disabling writes for close at 1731170941293Writing region close event to WAL at 1731170941294 (+1 ms)Closed at 1731170941294 2024-11-09T16:49:01,296 WARN [hfile-prefetch-1731170940261 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2/family/17c8e2c7523346868abe2d860a571a60, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-09T16:49:01,296 DEBUG [hfile-prefetch-1731170940261 {}] hfile.PrefetchExecutor(121): Prefetch completed for 17c8e2c7523346868abe2d860a571a60 2024-11-09T16:49:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-09T16:49:01,451 INFO [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:49:01,451 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-09T16:49:01,452 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 696e42658047076b8f9ad1d9849106e2, disabling compactions & flushes 2024-11-09T16:49:01,452 INFO [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:49:01,452 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:49:01,452 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. after waiting 0 ms 2024-11-09T16:49:01,452 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:49:01,454 DEBUG [StoreFileCloser-696e42658047076b8f9ad1d9849106e2-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file 17c8e2c7523346868abe2d860a571a60, starting offset: 0, end offset: 9223372036854775807 2024-11-09T16:49:01,455 DEBUG [StoreFileCloser-696e42658047076b8f9ad1d9849106e2-family-1 {}] bucket.BucketCache(857): removed key 17c8e2c7523346868abe2d860a571a60_0 from back map with offset lock 2094080 in the evict process 2024-11-09T16:49:01,458 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-09T16:49:01,458 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-09T16:49:01,458 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-09T16:49:01,458 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-09T16:49:01,459 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-09T16:49:01,459 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-09T16:49:01,459 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-09T16:49:01,459 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-09T16:49:01,461 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-09T16:49:01,462 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-09T16:49:01,462 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731170941458 2024-11-09T16:49:01,462 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-09T16:49:01,463 ERROR [bucket-cache-persister {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1679) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1497) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2024-11-09T16:49:01,463 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-09T16:49:01,463 INFO [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:49:01,462 WARN [Thread-92 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-11-09T16:49:01,463 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731170941459 2024-11-09T16:49:01,463 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-09T16:49:01,463 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 696e42658047076b8f9ad1d9849106e2: Waiting for close lock at 1731170941452Running coprocessor pre-close hooks at 1731170941452Disabling compacts and flushes for region at 1731170941452Disabling writes for close at 1731170941452Writing region close event to WAL at 1731170941456 (+4 ms)Running coprocessor post-close hooks at 1731170941462 (+6 ms)Closed at 1731170941463 (+1 ms) 2024-11-09T16:49:01,464 INFO [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 696e42658047076b8f9ad1d9849106e2 move to 95f00e0c2391,35523,1731170937067 record at close sequenceid=6 2024-11-09T16:49:01,467 INFO [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:49:01,467 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=696e42658047076b8f9ad1d9849106e2, regionState=CLOSED 2024-11-09T16:49:01,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 696e42658047076b8f9ad1d9849106e2, server=95f00e0c2391,35901,1731170937217 because future has completed 2024-11-09T16:49:01,473 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-09T16:49:01,474 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 696e42658047076b8f9ad1d9849106e2, server=95f00e0c2391,35901,1731170937217 in 179 msec 2024-11-09T16:49:01,474 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731170939448, region=696e42658047076b8f9ad1d9849106e2, REOPEN/MOVE; state=CLOSED, location=95f00e0c2391,35523,1731170937067; forceNewPlan=false, retain=false 2024-11-09T16:49:01,625 INFO [95f00e0c2391:43031 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-09T16:49:01,625 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=696e42658047076b8f9ad1d9849106e2, regionState=OPENING, regionLocation=95f00e0c2391,35523,1731170937067 2024-11-09T16:49:01,629 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731170939448, region=696e42658047076b8f9ad1d9849106e2, REOPEN/MOVE because future has completed 2024-11-09T16:49:01,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 696e42658047076b8f9ad1d9849106e2, server=95f00e0c2391,35523,1731170937067}] 2024-11-09T16:49:01,784 INFO [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:49:01,785 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 696e42658047076b8f9ad1d9849106e2, NAME => 'testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2.', STARTKEY => '', ENDKEY => ''} 2024-11-09T16:49:01,785 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731170939448 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:49:01,785 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T16:49:01,786 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:49:01,786 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:49:01,787 INFO [StoreOpener-696e42658047076b8f9ad1d9849106e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:49:01,788 INFO [StoreOpener-696e42658047076b8f9ad1d9849106e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 696e42658047076b8f9ad1d9849106e2 columnFamilyName family 2024-11-09T16:49:01,788 DEBUG [StoreOpener-696e42658047076b8f9ad1d9849106e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:49:01,790 DEBUG [StoreFileOpener-696e42658047076b8f9ad1d9849106e2-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2/family/17c8e2c7523346868abe2d860a571a60, delay=997 ms 2024-11-09T16:49:01,791 DEBUG [StoreOpener-696e42658047076b8f9ad1d9849106e2-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2/family/17c8e2c7523346868abe2d860a571a60 2024-11-09T16:49:01,792 INFO [StoreOpener-696e42658047076b8f9ad1d9849106e2-1 {}] regionserver.HStore(327): Store=696e42658047076b8f9ad1d9849106e2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T16:49:01,792 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:49:01,793 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2 2024-11-09T16:49:01,793 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2 2024-11-09T16:49:01,793 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:49:01,793 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:49:01,798 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:49:01,798 INFO [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 696e42658047076b8f9ad1d9849106e2; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64498011, jitterRate=-0.038904741406440735}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T16:49:01,798 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:49:01,799 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 696e42658047076b8f9ad1d9849106e2: Running coprocessor pre-open hook at 1731170941786Writing region info on filesystem at 1731170941786Initializing all the Stores at 1731170941786Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731170941786Cleaning up temporary data from old regions at 1731170941793 (+7 ms)Running coprocessor post-open hooks at 1731170941798 (+5 ms)Region opened successfully at 1731170941799 (+1 ms) 2024-11-09T16:49:01,800 INFO [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2., pid=9, masterSystemTime=1731170941782 2024-11-09T16:49:01,803 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:49:01,803 INFO [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:49:01,803 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=696e42658047076b8f9ad1d9849106e2, regionState=OPEN, openSeqNum=10, regionLocation=95f00e0c2391,35523,1731170937067 2024-11-09T16:49:01,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 696e42658047076b8f9ad1d9849106e2, server=95f00e0c2391,35523,1731170937067 because future has completed 2024-11-09T16:49:01,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-09T16:49:01,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 696e42658047076b8f9ad1d9849106e2, server=95f00e0c2391,35523,1731170937067 in 178 msec 2024-11-09T16:49:01,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731170939448, region=696e42658047076b8f9ad1d9849106e2, REOPEN/MOVE in 525 msec 2024-11-09T16:49:01,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-09T16:49:01,839 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T16:49:01,839 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T16:49:01,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T16:49:01,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T16:49:01,847 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T16:49:01,847 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-09T16:49:01,848 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=520236456, stopped=false 2024-11-09T16:49:01,848 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=95f00e0c2391,43031,1731170936251 2024-11-09T16:49:01,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T16:49:01,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T16:49:01,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:01,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:01,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T16:49:01,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:01,851 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T16:49:01,852 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T16:49:01,852 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T16:49:01,852 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T16:49:01,852 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T16:49:01,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T16:49:01,852 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T16:49:01,853 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '95f00e0c2391,35523,1731170937067' ***** 2024-11-09T16:49:01,853 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-09T16:49:01,853 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '95f00e0c2391,35901,1731170937217' ***** 2024-11-09T16:49:01,853 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-09T16:49:01,854 INFO [RS:0;95f00e0c2391:35523 {}] bucket.BucketCache(1739): Disabling cache 2024-11-09T16:49:01,854 INFO [RS:1;95f00e0c2391:35901 {}] bucket.BucketCache(1739): Disabling cache 2024-11-09T16:49:01,854 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-09T16:49:01,854 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-09T16:49:01,854 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-09T16:49:01,854 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-09T16:49:01,854 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-09T16:49:01,854 INFO [RS:1;95f00e0c2391:35901 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/bucket.persistence 2024-11-09T16:49:01,854 INFO [RS:0;95f00e0c2391:35523 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/bucket.persistence 2024-11-09T16:49:01,854 INFO [RS:0;95f00e0c2391:35523 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-09T16:49:01,854 INFO [regionserver/95f00e0c2391:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T16:49:01,854 INFO [regionserver/95f00e0c2391:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T16:49:01,855 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-09T16:49:01,855 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-09T16:49:01,855 INFO [RS:1;95f00e0c2391:35901 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-09T16:49:01,855 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-09T16:49:01,865 DEBUG [RS:0;95f00e0c2391:35523 {}] bucket.BucketCache(1488): Thread RS:0;95f00e0c2391:35523 started persisting bucket cache to file 2024-11-09T16:49:01,865 DEBUG [RS:0;95f00e0c2391:35523 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-09T16:49:01,865 DEBUG [RS:0;95f00e0c2391:35523 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-09T16:49:01,865 DEBUG [RS:1;95f00e0c2391:35901 {}] bucket.BucketCache(1488): Thread RS:1;95f00e0c2391:35901 started persisting bucket cache to file 2024-11-09T16:49:01,865 DEBUG [RS:1;95f00e0c2391:35901 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-09T16:49:01,865 DEBUG [RS:1;95f00e0c2391:35901 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-09T16:49:01,869 DEBUG [RS:0;95f00e0c2391:35523 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-09T16:49:01,869 DEBUG [RS:0;95f00e0c2391:35523 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731170941865 2024-11-09T16:49:01,869 DEBUG [RS:1;95f00e0c2391:35901 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-09T16:49:01,869 DEBUG [RS:1;95f00e0c2391:35901 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731170941865 2024-11-09T16:49:01,869 DEBUG [RS:0;95f00e0c2391:35523 {}] bucket.BucketCache(1510): Thread RS:0;95f00e0c2391:35523 finished persisting bucket cache to file, renaming 2024-11-09T16:49:01,869 DEBUG [RS:1;95f00e0c2391:35901 {}] bucket.BucketCache(1510): Thread RS:1;95f00e0c2391:35901 finished persisting bucket cache to file, renaming 2024-11-09T16:49:01,869 WARN [RS:1;95f00e0c2391:35901 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-09T16:49:01,869 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T16:49:01,869 INFO [RS:1;95f00e0c2391:35901 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T16:49:01,869 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T16:49:01,869 INFO [RS:0;95f00e0c2391:35523 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T16:49:01,869 INFO [RS:1;95f00e0c2391:35901 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T16:49:01,869 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T16:49:01,870 INFO [RS:1;95f00e0c2391:35901 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T16:49:01,870 INFO [RS:0;95f00e0c2391:35523 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T16:49:01,870 INFO [RS:1;95f00e0c2391:35901 {}] regionserver.HRegionServer(959): stopping server 95f00e0c2391,35901,1731170937217 2024-11-09T16:49:01,870 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.HRegionServer(3091): Received CLOSE for 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:49:01,870 INFO [RS:1;95f00e0c2391:35901 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T16:49:01,870 INFO [RS:1;95f00e0c2391:35901 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;95f00e0c2391:35901. 2024-11-09T16:49:01,870 DEBUG [RS:1;95f00e0c2391:35901 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T16:49:01,870 DEBUG [RS:1;95f00e0c2391:35901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T16:49:01,870 INFO [RS:1;95f00e0c2391:35901 {}] regionserver.HRegionServer(976): stopping server 95f00e0c2391,35901,1731170937217; all regions closed. 2024-11-09T16:49:01,870 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.HRegionServer(959): stopping server 95f00e0c2391,35523,1731170937067 2024-11-09T16:49:01,870 INFO [RS:0;95f00e0c2391:35523 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T16:49:01,870 INFO [RS:0;95f00e0c2391:35523 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;95f00e0c2391:35523. 2024-11-09T16:49:01,871 DEBUG [RS:0;95f00e0c2391:35523 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T16:49:01,871 DEBUG [RS:0;95f00e0c2391:35523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T16:49:01,871 INFO [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-09T16:49:01,871 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T16:49:01,871 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T16:49:01,871 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T16:49:01,871 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-09T16:49:01,871 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 696e42658047076b8f9ad1d9849106e2, disabling compactions & flushes 2024-11-09T16:49:01,871 INFO [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:49:01,871 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:49:01,872 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. after waiting 0 ms 2024-11-09T16:49:01,872 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-09T16:49:01,872 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:49:01,872 DEBUG [RS:0;95f00e0c2391:35523 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 696e42658047076b8f9ad1d9849106e2=testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2.} 2024-11-09T16:49:01,872 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-09T16:49:01,872 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T16:49:01,872 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T16:49:01,872 DEBUG [RS:0;95f00e0c2391:35523 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 696e42658047076b8f9ad1d9849106e2 2024-11-09T16:49:01,872 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T16:49:01,872 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T16:49:01,872 DEBUG [StoreFileCloser-696e42658047076b8f9ad1d9849106e2-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2/family/17c8e2c7523346868abe2d860a571a60 2024-11-09T16:49:01,872 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T16:49:01,873 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-09T16:49:01,876 DEBUG [RS:1;95f00e0c2391:35901 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/oldWALs 2024-11-09T16:49:01,876 INFO [RS:1;95f00e0c2391:35901 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 95f00e0c2391%2C35901%2C1731170937217:(num 1731170938735) 2024-11-09T16:49:01,876 DEBUG [RS:1;95f00e0c2391:35901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T16:49:01,876 INFO [RS:1;95f00e0c2391:35901 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T16:49:01,877 INFO [RS:1;95f00e0c2391:35901 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T16:49:01,877 INFO [RS:1;95f00e0c2391:35901 {}] hbase.ChoreService(370): Chore service for: regionserver/95f00e0c2391:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-09T16:49:01,877 INFO [RS:1;95f00e0c2391:35901 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T16:49:01,877 INFO [regionserver/95f00e0c2391:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T16:49:01,877 INFO [RS:1;95f00e0c2391:35901 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T16:49:01,877 INFO [RS:1;95f00e0c2391:35901 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T16:49:01,877 INFO [RS:1;95f00e0c2391:35901 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T16:49:01,878 INFO [RS:1;95f00e0c2391:35901 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35901 2024-11-09T16:49:01,878 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnRegionMove1731170939448/696e42658047076b8f9ad1d9849106e2/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-09T16:49:01,878 INFO [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:49:01,879 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 696e42658047076b8f9ad1d9849106e2: Waiting for close lock at 1731170941870Running coprocessor pre-close hooks at 1731170941871 (+1 ms)Disabling compacts and flushes for region at 1731170941871Disabling writes for close at 1731170941872 (+1 ms)Writing region close event to WAL at 1731170941873 (+1 ms)Running coprocessor post-close hooks at 1731170941878 (+5 ms)Closed at 1731170941878 2024-11-09T16:49:01,879 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2. 2024-11-09T16:49:01,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T16:49:01,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/95f00e0c2391,35901,1731170937217 2024-11-09T16:49:01,883 INFO [RS:1;95f00e0c2391:35901 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T16:49:01,886 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [95f00e0c2391,35901,1731170937217] 2024-11-09T16:49:01,886 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0f2ff24e7cdb40308d9b01fa29592846 2024-11-09T16:49:01,887 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/95f00e0c2391,35901,1731170937217 already deleted, retry=false 2024-11-09T16:49:01,887 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 95f00e0c2391,35901,1731170937217 expired; onlineServers=1 2024-11-09T16:49:01,905 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/.tmp/info/0f2ff24e7cdb40308d9b01fa29592846 is 205, key is testBlockEvictionOnRegionMove1731170939448,,1731170939467.696e42658047076b8f9ad1d9849106e2./info:regioninfo/1731170941803/Put/seqid=0 2024-11-09T16:49:01,906 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/.tmp/info/0f2ff24e7cdb40308d9b01fa29592846 2024-11-09T16:49:01,916 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2bd663d92050435a8aa7e818a02c4893 2024-11-09T16:49:01,931 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/.tmp/ns/2bd663d92050435a8aa7e818a02c4893 is 43, key is default/ns:d/1731170939183/Put/seqid=0 2024-11-09T16:49:01,932 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/.tmp/ns/2bd663d92050435a8aa7e818a02c4893 2024-11-09T16:49:01,942 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fbdf6278e8a24690923975bbbfea3fb3 2024-11-09T16:49:01,957 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/.tmp/table/fbdf6278e8a24690923975bbbfea3fb3 is 78, key is testBlockEvictionOnRegionMove1731170939448/table:state/1731170939876/Put/seqid=0 2024-11-09T16:49:01,958 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/.tmp/table/fbdf6278e8a24690923975bbbfea3fb3 2024-11-09T16:49:01,961 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/.tmp/info/0f2ff24e7cdb40308d9b01fa29592846 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/info/0f2ff24e7cdb40308d9b01fa29592846 2024-11-09T16:49:01,962 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/info/0f2ff24e7cdb40308d9b01fa29592846, delay=946 ms 2024-11-09T16:49:01,963 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/info/0f2ff24e7cdb40308d9b01fa29592846, entries=15, sequenceid=15, filesize=7.6 K 2024-11-09T16:49:01,964 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/.tmp/ns/2bd663d92050435a8aa7e818a02c4893 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/ns/2bd663d92050435a8aa7e818a02c4893 2024-11-09T16:49:01,965 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/ns/2bd663d92050435a8aa7e818a02c4893, delay=939 ms 2024-11-09T16:49:01,967 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/ns/2bd663d92050435a8aa7e818a02c4893, entries=2, sequenceid=15, filesize=5.0 K 2024-11-09T16:49:01,967 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/.tmp/table/fbdf6278e8a24690923975bbbfea3fb3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/table/fbdf6278e8a24690923975bbbfea3fb3 2024-11-09T16:49:01,968 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/table/fbdf6278e8a24690923975bbbfea3fb3, delay=977 ms 2024-11-09T16:49:01,970 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/table/fbdf6278e8a24690923975bbbfea3fb3, entries=2, sequenceid=15, filesize=5.3 K 2024-11-09T16:49:01,971 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 98ms, sequenceid=15, compaction requested=false 2024-11-09T16:49:01,971 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-09T16:49:01,972 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/info/0f2ff24e7cdb40308d9b01fa29592846 2024-11-09T16:49:01,973 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/ns/2bd663d92050435a8aa7e818a02c4893 2024-11-09T16:49:01,974 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/table/fbdf6278e8a24690923975bbbfea3fb3 2024-11-09T16:49:01,979 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-09T16:49:01,980 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T16:49:01,980 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T16:49:01,980 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731170941872Running coprocessor pre-close hooks at 1731170941872Disabling compacts and flushes for region at 1731170941872Disabling writes for close at 1731170941872Obtaining lock to block concurrent updates at 1731170941873 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731170941873Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731170941873Flushing stores of hbase:meta,,1.1588230740 at 1731170941875 (+2 ms)Flushing 1588230740/info: creating writer at 1731170941875Flushing 1588230740/info: appending metadata at 1731170941903 (+28 ms)Flushing 1588230740/info: closing flushed file at 1731170941903Flushing 1588230740/ns: creating writer at 1731170941909 (+6 ms)Flushing 1588230740/ns: appending metadata at 1731170941930 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1731170941930Flushing 1588230740/table: creating writer at 1731170941935 (+5 ms)Flushing 1588230740/table: appending metadata at 1731170941956 (+21 ms)Flushing 1588230740/table: closing flushed file at 1731170941956Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e58d1d0: reopening flushed file at 1731170941960 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ad8dc9d: reopening flushed file at 1731170941964 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42037f1a: reopening flushed file at 1731170941967 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 98ms, sequenceid=15, compaction requested=false at 1731170941971 (+4 ms)Writing region close event to WAL at 1731170941975 (+4 ms)Running coprocessor post-close hooks at 1731170941979 (+4 ms)Closed at 1731170941980 (+1 ms) 2024-11-09T16:49:01,981 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-09T16:49:01,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T16:49:01,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35901-0x1018def99e00002, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T16:49:01,986 INFO [RS:1;95f00e0c2391:35901 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T16:49:01,987 INFO [RS:1;95f00e0c2391:35901 {}] regionserver.HRegionServer(1031): Exiting; stopping=95f00e0c2391,35901,1731170937217; zookeeper connection closed. 2024-11-09T16:49:01,987 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1ff01c9a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1ff01c9a 2024-11-09T16:49:02,072 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.HRegionServer(976): stopping server 95f00e0c2391,35523,1731170937067; all regions closed. 2024-11-09T16:49:02,075 DEBUG [RS:0;95f00e0c2391:35523 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/oldWALs 2024-11-09T16:49:02,075 INFO [RS:0;95f00e0c2391:35523 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 95f00e0c2391%2C35523%2C1731170937067.meta:.meta(num 1731170939052) 2024-11-09T16:49:02,077 DEBUG [RS:0;95f00e0c2391:35523 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/oldWALs 2024-11-09T16:49:02,077 INFO [RS:0;95f00e0c2391:35523 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 95f00e0c2391%2C35523%2C1731170937067:(num 1731170938735) 2024-11-09T16:49:02,077 DEBUG [RS:0;95f00e0c2391:35523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T16:49:02,077 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T16:49:02,077 INFO [RS:0;95f00e0c2391:35523 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T16:49:02,077 INFO [RS:0;95f00e0c2391:35523 {}] hbase.ChoreService(370): Chore service for: regionserver/95f00e0c2391:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-09T16:49:02,077 INFO [RS:0;95f00e0c2391:35523 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T16:49:02,078 INFO [regionserver/95f00e0c2391:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T16:49:02,078 INFO [RS:0;95f00e0c2391:35523 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35523 2024-11-09T16:49:02,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/95f00e0c2391,35523,1731170937067 2024-11-09T16:49:02,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T16:49:02,080 INFO [RS:0;95f00e0c2391:35523 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T16:49:02,081 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [95f00e0c2391,35523,1731170937067] 2024-11-09T16:49:02,082 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/95f00e0c2391,35523,1731170937067 already deleted, retry=false 2024-11-09T16:49:02,082 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 95f00e0c2391,35523,1731170937067 expired; onlineServers=0 2024-11-09T16:49:02,083 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '95f00e0c2391,43031,1731170936251' ***** 2024-11-09T16:49:02,083 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-09T16:49:02,083 INFO [M:0;95f00e0c2391:43031 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T16:49:02,083 INFO [M:0;95f00e0c2391:43031 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T16:49:02,083 DEBUG [M:0;95f00e0c2391:43031 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-09T16:49:02,083 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-09T16:49:02,083 DEBUG [M:0;95f00e0c2391:43031 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-09T16:49:02,083 DEBUG [master/95f00e0c2391:0:becomeActiveMaster-HFileCleaner.large.0-1731170938323 {}] cleaner.HFileCleaner(306): Exit Thread[master/95f00e0c2391:0:becomeActiveMaster-HFileCleaner.large.0-1731170938323,5,FailOnTimeoutGroup] 2024-11-09T16:49:02,083 DEBUG [master/95f00e0c2391:0:becomeActiveMaster-HFileCleaner.small.0-1731170938323 {}] cleaner.HFileCleaner(306): Exit Thread[master/95f00e0c2391:0:becomeActiveMaster-HFileCleaner.small.0-1731170938323,5,FailOnTimeoutGroup] 2024-11-09T16:49:02,084 INFO [M:0;95f00e0c2391:43031 {}] hbase.ChoreService(370): Chore service for: master/95f00e0c2391:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-09T16:49:02,084 INFO [M:0;95f00e0c2391:43031 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T16:49:02,084 DEBUG [M:0;95f00e0c2391:43031 {}] master.HMaster(1795): Stopping service threads 2024-11-09T16:49:02,084 INFO [M:0;95f00e0c2391:43031 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-09T16:49:02,084 INFO [M:0;95f00e0c2391:43031 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T16:49:02,085 INFO [M:0;95f00e0c2391:43031 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T16:49:02,085 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-09T16:49:02,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-09T16:49:02,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:02,086 DEBUG [M:0;95f00e0c2391:43031 {}] zookeeper.ZKUtil(347): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-09T16:49:02,086 WARN [M:0;95f00e0c2391:43031 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-09T16:49:02,086 INFO [M:0;95f00e0c2391:43031 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/.lastflushedseqids 2024-11-09T16:49:02,099 INFO [M:0;95f00e0c2391:43031 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T16:49:02,099 INFO [M:0;95f00e0c2391:43031 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-09T16:49:02,100 DEBUG [M:0;95f00e0c2391:43031 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T16:49:02,100 INFO [M:0;95f00e0c2391:43031 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T16:49:02,100 DEBUG [M:0;95f00e0c2391:43031 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T16:49:02,100 DEBUG [M:0;95f00e0c2391:43031 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T16:49:02,100 DEBUG [M:0;95f00e0c2391:43031 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T16:49:02,100 INFO [M:0;95f00e0c2391:43031 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.66 KB heapSize=42.99 KB 2024-11-09T16:49:02,110 DEBUG [M:0;95f00e0c2391:43031 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c619209e0082455e9c46503652cccd57 2024-11-09T16:49:02,124 DEBUG [M:0;95f00e0c2391:43031 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c619209e0082455e9c46503652cccd57 is 82, key is hbase:meta,,1/info:regioninfo/1731170939118/Put/seqid=0 2024-11-09T16:49:02,125 INFO [M:0;95f00e0c2391:43031 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c619209e0082455e9c46503652cccd57 2024-11-09T16:49:02,133 DEBUG [M:0;95f00e0c2391:43031 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0d6eeadcd47d4ea4bfb63353fb5d70e7 2024-11-09T16:49:02,147 DEBUG [M:0;95f00e0c2391:43031 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0d6eeadcd47d4ea4bfb63353fb5d70e7 is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731170939882/Put/seqid=0 2024-11-09T16:49:02,148 INFO [M:0;95f00e0c2391:43031 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.04 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0d6eeadcd47d4ea4bfb63353fb5d70e7 2024-11-09T16:49:02,156 DEBUG [M:0;95f00e0c2391:43031 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 89ecbd8456744ae4bfcd10718f2654b7 2024-11-09T16:49:02,169 DEBUG [M:0;95f00e0c2391:43031 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/89ecbd8456744ae4bfcd10718f2654b7 is 69, key is 95f00e0c2391,35523,1731170937067/rs:state/1731170938462/Put/seqid=0 2024-11-09T16:49:02,169 INFO [M:0;95f00e0c2391:43031 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/89ecbd8456744ae4bfcd10718f2654b7 2024-11-09T16:49:02,171 DEBUG [M:0;95f00e0c2391:43031 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c619209e0082455e9c46503652cccd57 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c619209e0082455e9c46503652cccd57 2024-11-09T16:49:02,173 INFO [M:0;95f00e0c2391:43031 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c619209e0082455e9c46503652cccd57, entries=8, sequenceid=84, filesize=5.5 K 2024-11-09T16:49:02,173 DEBUG [M:0;95f00e0c2391:43031 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0d6eeadcd47d4ea4bfb63353fb5d70e7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0d6eeadcd47d4ea4bfb63353fb5d70e7 2024-11-09T16:49:02,175 INFO [M:0;95f00e0c2391:43031 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0d6eeadcd47d4ea4bfb63353fb5d70e7, entries=9, sequenceid=84, filesize=6.5 K 2024-11-09T16:49:02,175 DEBUG [M:0;95f00e0c2391:43031 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/89ecbd8456744ae4bfcd10718f2654b7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/89ecbd8456744ae4bfcd10718f2654b7 2024-11-09T16:49:02,177 INFO [M:0;95f00e0c2391:43031 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/89ecbd8456744ae4bfcd10718f2654b7, entries=2, sequenceid=84, filesize=5.1 K 2024-11-09T16:49:02,177 INFO [M:0;95f00e0c2391:43031 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 77ms, sequenceid=84, compaction requested=false 2024-11-09T16:49:02,179 INFO [M:0;95f00e0c2391:43031 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T16:49:02,179 DEBUG [M:0;95f00e0c2391:43031 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731170942100Disabling compacts and flushes for region at 1731170942100Disabling writes for close at 1731170942100Obtaining lock to block concurrent updates at 1731170942100Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731170942100Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35494, getHeapSize=43960, getOffHeapSize=0, getCellsCount=99 at 1731170942101 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731170942101Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731170942101Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731170942123 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731170942124 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731170942127 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731170942147 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731170942147Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731170942150 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731170942168 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731170942168Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22f62144: reopening flushed file at 1731170942171 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c4f835b: reopening flushed file at 1731170942173 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12f03042: reopening flushed file at 1731170942175 (+2 ms)Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 77ms, sequenceid=84, compaction requested=false at 1731170942177 (+2 ms)Writing region close event to WAL at 1731170942179 (+2 ms)Closed at 1731170942179 2024-11-09T16:49:02,181 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T16:49:02,181 INFO [M:0;95f00e0c2391:43031 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-09T16:49:02,181 INFO [M:0;95f00e0c2391:43031 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43031 2024-11-09T16:49:02,182 INFO [M:0;95f00e0c2391:43031 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T16:49:02,182 INFO [RS:0;95f00e0c2391:35523 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T16:49:02,182 INFO [RS:0;95f00e0c2391:35523 {}] regionserver.HRegionServer(1031): Exiting; stopping=95f00e0c2391,35523,1731170937067; zookeeper connection closed. 2024-11-09T16:49:02,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T16:49:02,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x1018def99e00001, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T16:49:02,183 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@639d44b0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@639d44b0 2024-11-09T16:49:02,183 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-09T16:49:02,284 INFO [M:0;95f00e0c2391:43031 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T16:49:02,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T16:49:02,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1018def99e00000, quorum=127.0.0.1:54118, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T16:49:02,289 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-09T16:49:02,310 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-09T16:49:02,310 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:49:02,318 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/95f00e0c2391:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1731170941790 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731170941965 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/95f00e0c2391:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/95f00e0c2391:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731170940261 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731170941962 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/95f00e0c2391:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=186 (was 166) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8720 (was 9080) 2024-11-09T16:49:02,325 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=186, ProcessCount=11, AvailableMemoryMB=8719 2024-11-09T16:49:02,325 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/hadoop-log-dir so I do NOT create it in target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e 2024-11-09T16:49:02,325 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/hadoop-tmp-dir so I do NOT create it in target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e 2024-11-09T16:49:02,325 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:49:02,328 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e/cluster_451fb58b-c0de-5278-9d18-fc958f6d9349, deleteOnExit=true 2024-11-09T16:49:02,331 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e/cluster_451fb58b-c0de-5278-9d18-fc958f6d9349/zookeeper_0, clientPort=51334, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e/cluster_451fb58b-c0de-5278-9d18-fc958f6d9349/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e/cluster_451fb58b-c0de-5278-9d18-fc958f6d9349/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-09T16:49:02,332 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51334 2024-11-09T16:49:02,344 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18 with version=8 2024-11-09T16:49:02,344 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/hbase-staging 2024-11-09T16:49:02,347 INFO [Time-limited test {}] client.ConnectionUtils(128): master/95f00e0c2391:0 server-side Connection retries=45 2024-11-09T16:49:02,347 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T16:49:02,347 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T16:49:02,348 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T16:49:02,348 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T16:49:02,348 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T16:49:02,348 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-09T16:49:02,348 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T16:49:02,349 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40569 2024-11-09T16:49:02,350 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40569 connecting to ZooKeeper ensemble=127.0.0.1:51334 2024-11-09T16:49:02,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:405690x0, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T16:49:02,356 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40569-0x1018defb2e30000 connected 2024-11-09T16:49:02,371 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:49:02,371 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:49:02,372 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T16:49:02,373 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18, hbase.cluster.distributed=false 2024-11-09T16:49:02,374 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T16:49:02,375 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40569 2024-11-09T16:49:02,375 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40569 2024-11-09T16:49:02,375 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40569 2024-11-09T16:49:02,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40569 2024-11-09T16:49:02,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40569 2024-11-09T16:49:02,392 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/95f00e0c2391:0 server-side Connection retries=45 2024-11-09T16:49:02,393 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T16:49:02,393 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T16:49:02,393 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T16:49:02,393 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T16:49:02,393 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T16:49:02,393 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T16:49:02,393 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T16:49:02,394 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37475 2024-11-09T16:49:02,395 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37475 connecting to ZooKeeper ensemble=127.0.0.1:51334 2024-11-09T16:49:02,395 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:49:02,396 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:49:02,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:374750x0, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T16:49:02,399 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37475-0x1018defb2e30001 connected 2024-11-09T16:49:02,399 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T16:49:02,399 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T16:49:02,400 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e/bucket.cache 2024-11-09T16:49:02,400 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-09T16:49:02,400 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-09T16:49:02,400 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-09T16:49:02,401 INFO [Thread-121 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-09T16:49:02,401 WARN [Thread-121 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-09T16:49:02,401 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-09T16:49:02,401 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T16:49:02,401 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-09T16:49:02,401 INFO [Thread-121 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e/bucket.persistence is complete. 2024-11-09T16:49:02,402 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T16:49:02,403 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T16:49:02,404 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37475 2024-11-09T16:49:02,404 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37475 2024-11-09T16:49:02,407 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37475 2024-11-09T16:49:02,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37475 2024-11-09T16:49:02,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37475 2024-11-09T16:49:02,424 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/95f00e0c2391:0 server-side Connection retries=45 2024-11-09T16:49:02,424 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T16:49:02,424 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T16:49:02,424 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T16:49:02,424 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T16:49:02,425 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T16:49:02,425 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T16:49:02,425 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T16:49:02,425 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34767 2024-11-09T16:49:02,427 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34767 connecting to ZooKeeper ensemble=127.0.0.1:51334 2024-11-09T16:49:02,427 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:49:02,427 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:49:02,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:347670x0, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T16:49:02,431 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34767-0x1018defb2e30002 connected 2024-11-09T16:49:02,431 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34767-0x1018defb2e30002, quorum=127.0.0.1:51334, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T16:49:02,431 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T16:49:02,432 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e/bucket.cache 2024-11-09T16:49:02,432 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-09T16:49:02,432 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-09T16:49:02,432 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-09T16:49:02,435 INFO [Thread-122 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-09T16:49:02,435 WARN [Thread-122 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-09T16:49:02,435 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-09T16:49:02,436 INFO [Thread-122 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-09T16:49:02,436 INFO [Thread-122 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e/bucket.persistence is complete. 2024-11-09T16:49:02,436 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T16:49:02,436 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34767-0x1018defb2e30002, quorum=127.0.0.1:51334, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T16:49:02,438 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34767-0x1018defb2e30002, quorum=127.0.0.1:51334, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T16:49:02,439 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34767 2024-11-09T16:49:02,439 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34767 2024-11-09T16:49:02,440 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34767 2024-11-09T16:49:02,440 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34767 2024-11-09T16:49:02,440 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34767 2024-11-09T16:49:02,461 DEBUG [M:0;95f00e0c2391:40569 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;95f00e0c2391:40569 2024-11-09T16:49:02,462 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/95f00e0c2391,40569,1731170942346 2024-11-09T16:49:02,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34767-0x1018defb2e30002, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T16:49:02,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T16:49:02,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T16:49:02,464 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/95f00e0c2391,40569,1731170942346 2024-11-09T16:49:02,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T16:49:02,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:02,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:02,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34767-0x1018defb2e30002, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T16:49:02,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34767-0x1018defb2e30002, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:02,467 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T16:49:02,468 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/95f00e0c2391,40569,1731170942346 from backup master directory 2024-11-09T16:49:02,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34767-0x1018defb2e30002, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T16:49:02,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/95f00e0c2391,40569,1731170942346 2024-11-09T16:49:02,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T16:49:02,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T16:49:02,469 WARN [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T16:49:02,470 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=95f00e0c2391,40569,1731170942346 2024-11-09T16:49:02,471 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/hbase.id] with ID: 4061dfbb-ef88-4578-93d7-484fbec30f9a 2024-11-09T16:49:02,471 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/.tmp/hbase.id 2024-11-09T16:49:02,475 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/hbase.id] 2024-11-09T16:49:02,510 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:49:02,510 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-09T16:49:02,512 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-09T16:49:02,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:02,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:02,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34767-0x1018defb2e30002, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:02,519 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T16:49:02,520 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-09T16:49:02,520 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T16:49:02,541 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store 2024-11-09T16:49:02,548 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T16:49:02,549 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T16:49:02,549 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T16:49:02,549 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T16:49:02,549 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T16:49:02,549 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T16:49:02,549 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T16:49:02,549 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731170942548Disabling compacts and flushes for region at 1731170942548Disabling writes for close at 1731170942549 (+1 ms)Writing region close event to WAL at 1731170942549Closed at 1731170942549 2024-11-09T16:49:02,549 WARN [master/95f00e0c2391:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/.initializing 2024-11-09T16:49:02,549 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/WALs/95f00e0c2391,40569,1731170942346 2024-11-09T16:49:02,554 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T16:49:02,558 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95f00e0c2391%2C40569%2C1731170942346, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/WALs/95f00e0c2391,40569,1731170942346, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/oldWALs, maxLogs=10 2024-11-09T16:49:02,576 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/WALs/95f00e0c2391,40569,1731170942346/95f00e0c2391%2C40569%2C1731170942346.1731170942559 2024-11-09T16:49:02,577 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-09T16:49:02,577 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-09T16:49:02,577 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T16:49:02,577 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:49:02,577 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:49:02,578 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:49:02,582 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-09T16:49:02,583 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:49:02,583 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T16:49:02,583 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:49:02,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-09T16:49:02,587 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:49:02,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T16:49:02,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:49:02,591 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-09T16:49:02,591 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:49:02,591 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T16:49:02,591 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:49:02,595 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-09T16:49:02,595 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:49:02,595 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T16:49:02,595 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:49:02,596 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:49:02,596 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:49:02,596 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:49:02,596 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:49:02,596 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T16:49:02,597 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T16:49:02,603 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T16:49:02,603 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74026744, jitterRate=0.10308444499969482}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T16:49:02,604 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731170942578Initializing all the Stores at 1731170942578Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731170942578Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731170942578Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731170942578Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731170942578Cleaning up temporary data from old regions at 1731170942596 (+18 ms)Region opened successfully at 1731170942604 (+8 ms) 2024-11-09T16:49:02,604 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-09T16:49:02,608 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69276f82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95f00e0c2391/172.17.0.2:0 2024-11-09T16:49:02,609 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-09T16:49:02,610 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-09T16:49:02,610 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-09T16:49:02,610 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T16:49:02,610 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T16:49:02,611 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-09T16:49:02,611 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-09T16:49:02,612 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-09T16:49:02,613 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-09T16:49:02,615 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-09T16:49:02,615 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-09T16:49:02,616 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-09T16:49:02,617 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-09T16:49:02,617 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-09T16:49:02,618 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-09T16:49:02,620 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-09T16:49:02,621 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-09T16:49:02,622 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-09T16:49:02,624 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-09T16:49:02,625 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-09T16:49:02,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T16:49:02,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T16:49:02,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:02,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:02,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34767-0x1018defb2e30002, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T16:49:02,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34767-0x1018defb2e30002, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:02,628 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=95f00e0c2391,40569,1731170942346, sessionid=0x1018defb2e30000, setting cluster-up flag (Was=false) 2024-11-09T16:49:02,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:02,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:02,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34767-0x1018defb2e30002, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:02,636 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-09T16:49:02,637 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=95f00e0c2391,40569,1731170942346 2024-11-09T16:49:02,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:02,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:02,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34767-0x1018defb2e30002, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:02,646 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-09T16:49:02,647 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=95f00e0c2391,40569,1731170942346 2024-11-09T16:49:02,648 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-09T16:49:02,650 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T16:49:02,650 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-09T16:49:02,651 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-09T16:49:02,651 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 95f00e0c2391,40569,1731170942346 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-09T16:49:02,653 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/95f00e0c2391:0, corePoolSize=5, maxPoolSize=5 2024-11-09T16:49:02,653 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/95f00e0c2391:0, corePoolSize=5, maxPoolSize=5 2024-11-09T16:49:02,653 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/95f00e0c2391:0, corePoolSize=5, maxPoolSize=5 2024-11-09T16:49:02,653 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/95f00e0c2391:0, corePoolSize=5, maxPoolSize=5 2024-11-09T16:49:02,653 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/95f00e0c2391:0, corePoolSize=10, maxPoolSize=10 2024-11-09T16:49:02,653 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,654 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/95f00e0c2391:0, corePoolSize=2, maxPoolSize=2 2024-11-09T16:49:02,654 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,655 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T16:49:02,655 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731170972655 2024-11-09T16:49:02,655 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-09T16:49:02,656 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-09T16:49:02,656 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-09T16:49:02,656 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-09T16:49:02,656 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-09T16:49:02,656 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-09T16:49:02,656 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:49:02,656 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-09T16:49:02,656 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,656 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-09T16:49:02,657 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-09T16:49:02,657 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-09T16:49:02,657 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-09T16:49:02,659 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-09T16:49:02,660 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-09T16:49:02,660 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/95f00e0c2391:0:becomeActiveMaster-HFileCleaner.large.0-1731170942660,5,FailOnTimeoutGroup] 2024-11-09T16:49:02,660 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/95f00e0c2391:0:becomeActiveMaster-HFileCleaner.small.0-1731170942660,5,FailOnTimeoutGroup] 2024-11-09T16:49:02,660 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,661 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-09T16:49:02,661 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,661 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,671 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-09T16:49:02,671 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18 2024-11-09T16:49:02,677 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T16:49:02,678 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-09T16:49:02,681 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T16:49:02,682 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:49:02,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T16:49:02,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-09T16:49:02,685 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T16:49:02,685 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:49:02,685 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T16:49:02,685 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-09T16:49:02,689 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T16:49:02,689 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:49:02,689 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T16:49:02,689 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-09T16:49:02,692 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T16:49:02,692 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:49:02,692 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T16:49:02,693 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T16:49:02,693 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740 2024-11-09T16:49:02,693 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740 2024-11-09T16:49:02,693 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T16:49:02,693 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T16:49:02,694 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T16:49:02,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T16:49:02,700 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T16:49:02,701 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64238807, jitterRate=-0.04276718199253082}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T16:49:02,702 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731170942678Initializing all the Stores at 1731170942678Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731170942678Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731170942678Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731170942678Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731170942678Cleaning up temporary data from old regions at 1731170942694 (+16 ms)Region opened successfully at 1731170942702 (+8 ms) 2024-11-09T16:49:02,702 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T16:49:02,702 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T16:49:02,702 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T16:49:02,702 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T16:49:02,702 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T16:49:02,703 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T16:49:02,703 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731170942702Disabling compacts and flushes for region at 1731170942702Disabling writes for close at 1731170942702Writing region close event to WAL at 1731170942703 (+1 ms)Closed at 1731170942703 2024-11-09T16:49:02,704 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T16:49:02,704 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-09T16:49:02,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T16:49:02,706 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T16:49:02,707 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-09T16:49:02,745 INFO [RS:1;95f00e0c2391:34767 {}] regionserver.HRegionServer(746): ClusterId : 4061dfbb-ef88-4578-93d7-484fbec30f9a 2024-11-09T16:49:02,745 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.HRegionServer(746): ClusterId : 4061dfbb-ef88-4578-93d7-484fbec30f9a 2024-11-09T16:49:02,745 DEBUG [RS:1;95f00e0c2391:34767 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T16:49:02,745 DEBUG [RS:0;95f00e0c2391:37475 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T16:49:02,747 DEBUG [RS:1;95f00e0c2391:34767 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T16:49:02,747 DEBUG [RS:0;95f00e0c2391:37475 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T16:49:02,747 DEBUG [RS:1;95f00e0c2391:34767 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T16:49:02,748 DEBUG [RS:0;95f00e0c2391:37475 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T16:49:02,750 DEBUG [RS:1;95f00e0c2391:34767 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T16:49:02,750 DEBUG [RS:0;95f00e0c2391:37475 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T16:49:02,750 DEBUG [RS:1;95f00e0c2391:34767 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c6b0106, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95f00e0c2391/172.17.0.2:0 2024-11-09T16:49:02,751 DEBUG [RS:0;95f00e0c2391:37475 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1544efaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95f00e0c2391/172.17.0.2:0 2024-11-09T16:49:02,764 DEBUG [RS:1;95f00e0c2391:34767 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;95f00e0c2391:34767 2024-11-09T16:49:02,764 INFO [RS:1;95f00e0c2391:34767 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T16:49:02,765 INFO [RS:1;95f00e0c2391:34767 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T16:49:02,765 DEBUG [RS:1;95f00e0c2391:34767 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T16:49:02,766 INFO [RS:1;95f00e0c2391:34767 {}] regionserver.HRegionServer(2659): reportForDuty to master=95f00e0c2391,40569,1731170942346 with port=34767, startcode=1731170942424 2024-11-09T16:49:02,766 DEBUG [RS:1;95f00e0c2391:34767 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T16:49:02,769 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41057, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T16:49:02,770 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40569 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 95f00e0c2391,34767,1731170942424 2024-11-09T16:49:02,770 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40569 {}] master.ServerManager(517): Registering regionserver=95f00e0c2391,34767,1731170942424 2024-11-09T16:49:02,770 DEBUG [RS:0;95f00e0c2391:37475 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;95f00e0c2391:37475 2024-11-09T16:49:02,770 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T16:49:02,770 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T16:49:02,770 DEBUG [RS:0;95f00e0c2391:37475 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T16:49:02,771 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.HRegionServer(2659): reportForDuty to master=95f00e0c2391,40569,1731170942346 with port=37475, startcode=1731170942392 2024-11-09T16:49:02,771 DEBUG [RS:0;95f00e0c2391:37475 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T16:49:02,771 DEBUG [RS:1;95f00e0c2391:34767 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18 2024-11-09T16:49:02,771 DEBUG [RS:1;95f00e0c2391:34767 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-09T16:49:02,772 DEBUG [RS:1;95f00e0c2391:34767 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T16:49:02,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T16:49:02,774 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38797, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T16:49:02,774 DEBUG [RS:1;95f00e0c2391:34767 {}] zookeeper.ZKUtil(111): regionserver:34767-0x1018defb2e30002, quorum=127.0.0.1:51334, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/95f00e0c2391,34767,1731170942424 2024-11-09T16:49:02,774 WARN [RS:1;95f00e0c2391:34767 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T16:49:02,774 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40569 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 95f00e0c2391,37475,1731170942392 2024-11-09T16:49:02,774 INFO [RS:1;95f00e0c2391:34767 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T16:49:02,774 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40569 {}] master.ServerManager(517): Registering regionserver=95f00e0c2391,37475,1731170942392 2024-11-09T16:49:02,775 DEBUG [RS:1;95f00e0c2391:34767 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,34767,1731170942424 2024-11-09T16:49:02,775 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [95f00e0c2391,34767,1731170942424] 2024-11-09T16:49:02,776 DEBUG [RS:0;95f00e0c2391:37475 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18 2024-11-09T16:49:02,776 DEBUG [RS:0;95f00e0c2391:37475 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-09T16:49:02,776 DEBUG [RS:0;95f00e0c2391:37475 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T16:49:02,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T16:49:02,781 DEBUG [RS:0;95f00e0c2391:37475 {}] zookeeper.ZKUtil(111): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/95f00e0c2391,37475,1731170942392 2024-11-09T16:49:02,781 WARN [RS:0;95f00e0c2391:37475 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T16:49:02,781 INFO [RS:0;95f00e0c2391:37475 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T16:49:02,781 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [95f00e0c2391,37475,1731170942392] 2024-11-09T16:49:02,781 DEBUG [RS:0;95f00e0c2391:37475 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,37475,1731170942392 2024-11-09T16:49:02,782 INFO [RS:1;95f00e0c2391:34767 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T16:49:02,785 INFO [RS:1;95f00e0c2391:34767 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T16:49:02,785 INFO [RS:1;95f00e0c2391:34767 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T16:49:02,785 INFO [RS:1;95f00e0c2391:34767 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,785 INFO [RS:1;95f00e0c2391:34767 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T16:49:02,786 INFO [RS:1;95f00e0c2391:34767 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T16:49:02,786 INFO [RS:1;95f00e0c2391:34767 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,786 DEBUG [RS:1;95f00e0c2391:34767 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,786 DEBUG [RS:1;95f00e0c2391:34767 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,787 DEBUG [RS:1;95f00e0c2391:34767 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,787 DEBUG [RS:1;95f00e0c2391:34767 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,787 DEBUG [RS:1;95f00e0c2391:34767 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,787 DEBUG [RS:1;95f00e0c2391:34767 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/95f00e0c2391:0, corePoolSize=2, maxPoolSize=2 2024-11-09T16:49:02,787 DEBUG [RS:1;95f00e0c2391:34767 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,787 DEBUG [RS:1;95f00e0c2391:34767 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,787 DEBUG [RS:1;95f00e0c2391:34767 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,787 DEBUG [RS:1;95f00e0c2391:34767 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,787 DEBUG [RS:1;95f00e0c2391:34767 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,787 DEBUG [RS:1;95f00e0c2391:34767 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,787 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T16:49:02,787 DEBUG [RS:1;95f00e0c2391:34767 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/95f00e0c2391:0, corePoolSize=3, maxPoolSize=3 2024-11-09T16:49:02,787 DEBUG [RS:1;95f00e0c2391:34767 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/95f00e0c2391:0, corePoolSize=3, maxPoolSize=3 2024-11-09T16:49:02,791 INFO [RS:1;95f00e0c2391:34767 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,791 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T16:49:02,791 INFO [RS:1;95f00e0c2391:34767 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,791 INFO [RS:1;95f00e0c2391:34767 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,791 INFO [RS:1;95f00e0c2391:34767 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,791 INFO [RS:1;95f00e0c2391:34767 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,791 INFO [RS:1;95f00e0c2391:34767 {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,34767,1731170942424-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T16:49:02,792 INFO [RS:0;95f00e0c2391:37475 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T16:49:02,792 INFO [RS:0;95f00e0c2391:37475 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,792 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T16:49:02,792 INFO [RS:0;95f00e0c2391:37475 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T16:49:02,793 INFO [RS:0;95f00e0c2391:37475 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,793 DEBUG [RS:0;95f00e0c2391:37475 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,793 DEBUG [RS:0;95f00e0c2391:37475 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,793 DEBUG [RS:0;95f00e0c2391:37475 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,793 DEBUG [RS:0;95f00e0c2391:37475 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,793 DEBUG [RS:0;95f00e0c2391:37475 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,793 DEBUG [RS:0;95f00e0c2391:37475 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/95f00e0c2391:0, corePoolSize=2, maxPoolSize=2 2024-11-09T16:49:02,794 DEBUG [RS:0;95f00e0c2391:37475 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,794 DEBUG [RS:0;95f00e0c2391:37475 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,794 DEBUG [RS:0;95f00e0c2391:37475 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,794 DEBUG [RS:0;95f00e0c2391:37475 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,794 DEBUG [RS:0;95f00e0c2391:37475 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,794 DEBUG [RS:0;95f00e0c2391:37475 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:02,794 DEBUG [RS:0;95f00e0c2391:37475 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/95f00e0c2391:0, corePoolSize=3, maxPoolSize=3 2024-11-09T16:49:02,794 DEBUG [RS:0;95f00e0c2391:37475 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/95f00e0c2391:0, corePoolSize=3, maxPoolSize=3 2024-11-09T16:49:02,796 INFO [RS:0;95f00e0c2391:37475 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,796 INFO [RS:0;95f00e0c2391:37475 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,796 INFO [RS:0;95f00e0c2391:37475 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,796 INFO [RS:0;95f00e0c2391:37475 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,796 INFO [RS:0;95f00e0c2391:37475 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,796 INFO [RS:0;95f00e0c2391:37475 {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,37475,1731170942392-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T16:49:02,808 INFO [RS:1;95f00e0c2391:34767 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T16:49:02,808 INFO [RS:1;95f00e0c2391:34767 {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,34767,1731170942424-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,809 INFO [RS:1;95f00e0c2391:34767 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,809 INFO [RS:1;95f00e0c2391:34767 {}] regionserver.Replication(171): 95f00e0c2391,34767,1731170942424 started 2024-11-09T16:49:02,820 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T16:49:02,820 INFO [RS:0;95f00e0c2391:37475 {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,37475,1731170942392-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,820 INFO [RS:0;95f00e0c2391:37475 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,820 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.Replication(171): 95f00e0c2391,37475,1731170942392 started 2024-11-09T16:49:02,824 INFO [RS:1;95f00e0c2391:34767 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,825 INFO [RS:1;95f00e0c2391:34767 {}] regionserver.HRegionServer(1482): Serving as 95f00e0c2391,34767,1731170942424, RpcServer on 95f00e0c2391/172.17.0.2:34767, sessionid=0x1018defb2e30002 2024-11-09T16:49:02,825 DEBUG [RS:1;95f00e0c2391:34767 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T16:49:02,825 DEBUG [RS:1;95f00e0c2391:34767 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 95f00e0c2391,34767,1731170942424 2024-11-09T16:49:02,825 DEBUG [RS:1;95f00e0c2391:34767 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95f00e0c2391,34767,1731170942424' 2024-11-09T16:49:02,825 DEBUG [RS:1;95f00e0c2391:34767 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T16:49:02,826 DEBUG [RS:1;95f00e0c2391:34767 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T16:49:02,826 DEBUG [RS:1;95f00e0c2391:34767 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T16:49:02,826 DEBUG [RS:1;95f00e0c2391:34767 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T16:49:02,826 DEBUG [RS:1;95f00e0c2391:34767 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 95f00e0c2391,34767,1731170942424 2024-11-09T16:49:02,826 DEBUG [RS:1;95f00e0c2391:34767 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95f00e0c2391,34767,1731170942424' 2024-11-09T16:49:02,826 DEBUG [RS:1;95f00e0c2391:34767 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T16:49:02,827 DEBUG [RS:1;95f00e0c2391:34767 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T16:49:02,827 DEBUG [RS:1;95f00e0c2391:34767 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T16:49:02,827 INFO [RS:1;95f00e0c2391:34767 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T16:49:02,827 INFO [RS:1;95f00e0c2391:34767 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T16:49:02,835 INFO [RS:0;95f00e0c2391:37475 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:02,836 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.HRegionServer(1482): Serving as 95f00e0c2391,37475,1731170942392, RpcServer on 95f00e0c2391/172.17.0.2:37475, sessionid=0x1018defb2e30001 2024-11-09T16:49:02,836 DEBUG [RS:0;95f00e0c2391:37475 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T16:49:02,836 DEBUG [RS:0;95f00e0c2391:37475 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 95f00e0c2391,37475,1731170942392 2024-11-09T16:49:02,836 DEBUG [RS:0;95f00e0c2391:37475 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95f00e0c2391,37475,1731170942392' 2024-11-09T16:49:02,836 DEBUG [RS:0;95f00e0c2391:37475 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T16:49:02,836 DEBUG [RS:0;95f00e0c2391:37475 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T16:49:02,837 DEBUG [RS:0;95f00e0c2391:37475 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T16:49:02,837 DEBUG [RS:0;95f00e0c2391:37475 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T16:49:02,837 DEBUG [RS:0;95f00e0c2391:37475 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 95f00e0c2391,37475,1731170942392 2024-11-09T16:49:02,837 DEBUG [RS:0;95f00e0c2391:37475 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95f00e0c2391,37475,1731170942392' 2024-11-09T16:49:02,837 DEBUG [RS:0;95f00e0c2391:37475 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T16:49:02,838 DEBUG [RS:0;95f00e0c2391:37475 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T16:49:02,838 DEBUG [RS:0;95f00e0c2391:37475 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T16:49:02,838 INFO [RS:0;95f00e0c2391:37475 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T16:49:02,838 INFO [RS:0;95f00e0c2391:37475 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T16:49:02,857 WARN [95f00e0c2391:40569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T16:49:02,928 INFO [RS:1;95f00e0c2391:34767 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T16:49:02,929 INFO [RS:1;95f00e0c2391:34767 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=95f00e0c2391%2C34767%2C1731170942424, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,34767,1731170942424, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/oldWALs, maxLogs=55 2024-11-09T16:49:02,939 INFO [RS:0;95f00e0c2391:37475 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T16:49:02,939 INFO [RS:0;95f00e0c2391:37475 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=95f00e0c2391%2C37475%2C1731170942392, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,37475,1731170942392, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/oldWALs, maxLogs=55 2024-11-09T16:49:02,950 INFO [RS:1;95f00e0c2391:34767 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,34767,1731170942424/95f00e0c2391%2C34767%2C1731170942424.1731170942931 2024-11-09T16:49:02,950 DEBUG [RS:1;95f00e0c2391:34767 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-09T16:49:02,960 INFO [RS:0;95f00e0c2391:37475 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,37475,1731170942392/95f00e0c2391%2C37475%2C1731170942392.1731170942941 2024-11-09T16:49:02,960 DEBUG [RS:0;95f00e0c2391:37475 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-09T16:49:03,107 DEBUG [95f00e0c2391:40569 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-09T16:49:03,108 DEBUG [95f00e0c2391:40569 {}] balancer.BalancerClusterState(204): Hosts are {95f00e0c2391=0} racks are {/default-rack=0} 2024-11-09T16:49:03,108 DEBUG [95f00e0c2391:40569 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-09T16:49:03,109 DEBUG [95f00e0c2391:40569 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-09T16:49:03,109 DEBUG [95f00e0c2391:40569 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-09T16:49:03,109 DEBUG [95f00e0c2391:40569 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-09T16:49:03,109 INFO [95f00e0c2391:40569 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-09T16:49:03,109 INFO [95f00e0c2391:40569 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-09T16:49:03,109 DEBUG [95f00e0c2391:40569 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-09T16:49:03,109 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=95f00e0c2391,37475,1731170942392 2024-11-09T16:49:03,110 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 95f00e0c2391,37475,1731170942392, state=OPENING 2024-11-09T16:49:03,112 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-09T16:49:03,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:03,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:03,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34767-0x1018defb2e30002, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:03,114 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T16:49:03,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=95f00e0c2391,37475,1731170942392}] 2024-11-09T16:49:03,115 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T16:49:03,115 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T16:49:03,115 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T16:49:03,268 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T16:49:03,270 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35479, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T16:49:03,272 INFO [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-09T16:49:03,272 INFO [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T16:49:03,273 INFO [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-09T16:49:03,274 INFO [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=95f00e0c2391%2C37475%2C1731170942392.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,37475,1731170942392, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/oldWALs, maxLogs=55 2024-11-09T16:49:03,301 INFO [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,37475,1731170942392/95f00e0c2391%2C37475%2C1731170942392.meta.1731170943275.meta 2024-11-09T16:49:03,301 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-09T16:49:03,302 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-09T16:49:03,302 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-09T16:49:03,302 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-09T16:49:03,303 INFO [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-09T16:49:03,303 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-09T16:49:03,303 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T16:49:03,303 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-09T16:49:03,303 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-09T16:49:03,304 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-09T16:49:03,305 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T16:49:03,305 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:49:03,305 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T16:49:03,306 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-09T16:49:03,306 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T16:49:03,306 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:49:03,306 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T16:49:03,307 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-09T16:49:03,307 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T16:49:03,307 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:49:03,307 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T16:49:03,308 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-09T16:49:03,308 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T16:49:03,308 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:49:03,308 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T16:49:03,308 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T16:49:03,309 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740 2024-11-09T16:49:03,309 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740 2024-11-09T16:49:03,310 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T16:49:03,310 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T16:49:03,310 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T16:49:03,311 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T16:49:03,311 INFO [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60537411, jitterRate=-0.09792228043079376}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T16:49:03,311 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-09T16:49:03,312 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731170943304Writing region info on filesystem at 1731170943304Initializing all the Stores at 1731170943304Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731170943304Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731170943304Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731170943304Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731170943304Cleaning up temporary data from old regions at 1731170943310 (+6 ms)Running coprocessor post-open hooks at 1731170943311 (+1 ms)Region opened successfully at 1731170943312 (+1 ms) 2024-11-09T16:49:03,312 INFO [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731170943267 2024-11-09T16:49:03,315 DEBUG [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-09T16:49:03,315 INFO [RS_OPEN_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-09T16:49:03,316 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=95f00e0c2391,37475,1731170942392 2024-11-09T16:49:03,316 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 95f00e0c2391,37475,1731170942392, state=OPEN 2024-11-09T16:49:03,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T16:49:03,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T16:49:03,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34767-0x1018defb2e30002, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T16:49:03,318 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T16:49:03,318 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T16:49:03,318 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=95f00e0c2391,37475,1731170942392 2024-11-09T16:49:03,318 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T16:49:03,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-09T16:49:03,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=95f00e0c2391,37475,1731170942392 in 205 msec 2024-11-09T16:49:03,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-09T16:49:03,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 616 msec 2024-11-09T16:49:03,324 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T16:49:03,324 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-09T16:49:03,328 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T16:49:03,328 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=95f00e0c2391,37475,1731170942392, seqNum=-1] 2024-11-09T16:49:03,329 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T16:49:03,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42421, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T16:49:03,336 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 686 msec 2024-11-09T16:49:03,337 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731170943336, completionTime=-1 2024-11-09T16:49:03,337 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-09T16:49:03,337 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T16:49:03,340 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T16:49:03,340 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731171003340 2024-11-09T16:49:03,340 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731171063340 2024-11-09T16:49:03,340 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-09T16:49:03,341 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,40569,1731170942346-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:03,341 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,40569,1731170942346-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:03,341 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,40569,1731170942346-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:03,341 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-95f00e0c2391:40569, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:03,341 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:03,342 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:03,344 DEBUG [master/95f00e0c2391:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-09T16:49:03,348 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.878sec 2024-11-09T16:49:03,348 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-09T16:49:03,348 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-09T16:49:03,348 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-09T16:49:03,348 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-09T16:49:03,348 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-09T16:49:03,348 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,40569,1731170942346-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T16:49:03,348 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,40569,1731170942346-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-09T16:49:03,349 DEBUG [master/95f00e0c2391:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-09T16:49:03,350 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-09T16:49:03,350 INFO [master/95f00e0c2391:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,40569,1731170942346-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:03,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7559f21a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T16:49:03,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 95f00e0c2391,40569,-1 for getting cluster id 2024-11-09T16:49:03,445 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T16:49:03,447 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4061dfbb-ef88-4578-93d7-484fbec30f9a' 2024-11-09T16:49:03,448 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T16:49:03,448 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4061dfbb-ef88-4578-93d7-484fbec30f9a" 2024-11-09T16:49:03,448 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bfd7f5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T16:49:03,448 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [95f00e0c2391,40569,-1] 2024-11-09T16:49:03,449 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T16:49:03,449 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T16:49:03,450 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T16:49:03,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27316b7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T16:49:03,452 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T16:49:03,454 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=95f00e0c2391,37475,1731170942392, seqNum=-1] 2024-11-09T16:49:03,454 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T16:49:03,456 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T16:49:03,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=95f00e0c2391,40569,1731170942346 2024-11-09T16:49:03,458 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T16:49:03,460 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 95f00e0c2391,40569,1731170942346 2024-11-09T16:49:03,460 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@687e6075 2024-11-09T16:49:03,460 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T16:49:03,462 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T16:49:03,463 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40569 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731170943458', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T16:49:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40569 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731170943458 2024-11-09T16:49:03,465 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731170943458 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T16:49:03,465 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:49:03,465 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40569 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731170943458" procId is: 4 2024-11-09T16:49:03,466 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731170943458 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T16:49:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T16:49:03,477 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731170943458-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7228d40e5e10b11f70426f05f75ec349, NAME => 'testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731170943458', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18 2024-11-09T16:49:03,483 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731170943458-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T16:49:03,483 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731170943458-pool-0 {}] regionserver.HRegion(1722): Closing 7228d40e5e10b11f70426f05f75ec349, disabling compactions & flushes 2024-11-09T16:49:03,483 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731170943458-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. 2024-11-09T16:49:03,483 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731170943458-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. 2024-11-09T16:49:03,483 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731170943458-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. after waiting 0 ms 2024-11-09T16:49:03,483 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731170943458-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. 2024-11-09T16:49:03,484 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731170943458-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. 2024-11-09T16:49:03,484 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731170943458-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7228d40e5e10b11f70426f05f75ec349: Waiting for close lock at 1731170943483Disabling compacts and flushes for region at 1731170943483Disabling writes for close at 1731170943483Writing region close event to WAL at 1731170943484 (+1 ms)Closed at 1731170943484 2024-11-09T16:49:03,485 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731170943458 execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T16:49:03,485 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731170943485"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731170943485"}]},"ts":"1731170943485"} 2024-11-09T16:49:03,488 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T16:49:03,488 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731170943458 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T16:49:03,489 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731170943458","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731170943489"}]},"ts":"1731170943489"} 2024-11-09T16:49:03,491 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731170943458, state=ENABLING in hbase:meta 2024-11-09T16:49:03,491 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {95f00e0c2391=0} racks are {/default-rack=0} 2024-11-09T16:49:03,492 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-09T16:49:03,492 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-09T16:49:03,492 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-09T16:49:03,492 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-09T16:49:03,492 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-09T16:49:03,492 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-09T16:49:03,492 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-09T16:49:03,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731170943458, region=7228d40e5e10b11f70426f05f75ec349, ASSIGN}] 2024-11-09T16:49:03,494 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731170943458, region=7228d40e5e10b11f70426f05f75ec349, ASSIGN 2024-11-09T16:49:03,494 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731170943458, region=7228d40e5e10b11f70426f05f75ec349, ASSIGN; state=OFFLINE, location=95f00e0c2391,34767,1731170942424; forceNewPlan=false, retain=false 2024-11-09T16:49:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T16:49:03,645 INFO [95f00e0c2391:40569 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-09T16:49:03,646 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7228d40e5e10b11f70426f05f75ec349, regionState=OPENING, regionLocation=95f00e0c2391,34767,1731170942424 2024-11-09T16:49:03,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731170943458, region=7228d40e5e10b11f70426f05f75ec349, ASSIGN because future has completed 2024-11-09T16:49:03,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7228d40e5e10b11f70426f05f75ec349, server=95f00e0c2391,34767,1731170942424}] 2024-11-09T16:49:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T16:49:03,802 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T16:49:03,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59363, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T16:49:03,806 INFO [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. 2024-11-09T16:49:03,806 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7228d40e5e10b11f70426f05f75ec349, NAME => 'testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349.', STARTKEY => '', ENDKEY => ''} 2024-11-09T16:49:03,806 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731170943458 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:03,806 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T16:49:03,807 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:03,807 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:03,807 INFO [StoreOpener-7228d40e5e10b11f70426f05f75ec349-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:03,811 INFO [StoreOpener-7228d40e5e10b11f70426f05f75ec349-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7228d40e5e10b11f70426f05f75ec349 columnFamilyName family 2024-11-09T16:49:03,812 DEBUG [StoreOpener-7228d40e5e10b11f70426f05f75ec349-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:49:03,812 INFO [StoreOpener-7228d40e5e10b11f70426f05f75ec349-1 {}] regionserver.HStore(327): Store=7228d40e5e10b11f70426f05f75ec349/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T16:49:03,812 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:03,813 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:03,813 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:03,813 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:03,813 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:03,814 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:03,820 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T16:49:03,820 INFO [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7228d40e5e10b11f70426f05f75ec349; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62760370, jitterRate=-0.06479761004447937}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T16:49:03,820 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:03,821 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7228d40e5e10b11f70426f05f75ec349: Running coprocessor pre-open hook at 1731170943807Writing region info on filesystem at 1731170943807Initializing all the Stores at 1731170943807Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731170943807Cleaning up temporary data from old regions at 1731170943813 (+6 ms)Running coprocessor post-open hooks at 1731170943820 (+7 ms)Region opened successfully at 1731170943821 (+1 ms) 2024-11-09T16:49:03,821 INFO [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349., pid=6, masterSystemTime=1731170943802 2024-11-09T16:49:03,824 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. 2024-11-09T16:49:03,824 INFO [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. 2024-11-09T16:49:03,825 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7228d40e5e10b11f70426f05f75ec349, regionState=OPEN, openSeqNum=2, regionLocation=95f00e0c2391,34767,1731170942424 2024-11-09T16:49:03,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7228d40e5e10b11f70426f05f75ec349, server=95f00e0c2391,34767,1731170942424 because future has completed 2024-11-09T16:49:03,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-09T16:49:03,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7228d40e5e10b11f70426f05f75ec349, server=95f00e0c2391,34767,1731170942424 in 180 msec 2024-11-09T16:49:03,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-09T16:49:03,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731170943458, region=7228d40e5e10b11f70426f05f75ec349, ASSIGN in 338 msec 2024-11-09T16:49:03,833 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731170943458 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T16:49:03,833 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731170943458","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731170943833"}]},"ts":"1731170943833"} 2024-11-09T16:49:03,835 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731170943458, state=ENABLED in hbase:meta 2024-11-09T16:49:03,836 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731170943458 execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T16:49:03,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731170943458 in 373 msec 2024-11-09T16:49:04,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T16:49:04,090 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731170943458 completed 2024-11-09T16:49:04,090 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731170943458 get assigned. Timeout = 60000ms 2024-11-09T16:49:04,091 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T16:49:04,095 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731170943458 assigned to meta. Checking AM states. 2024-11-09T16:49:04,095 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T16:49:04,095 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731170943458 assigned. 2024-11-09T16:49:04,098 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731170943458', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349., hostname=95f00e0c2391,34767,1731170942424, seqNum=2] 2024-11-09T16:49:04,099 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T16:49:04,101 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T16:49:04,105 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7228d40e5e10b11f70426f05f75ec349 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-09T16:49:04,116 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 591230670a164f54a3ce4eead06b241f 2024-11-09T16:49:04,130 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349/.tmp/family/591230670a164f54a3ce4eead06b241f is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-09T16:49:04,130 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349/.tmp/family/591230670a164f54a3ce4eead06b241f 2024-11-09T16:49:04,133 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349/.tmp/family/591230670a164f54a3ce4eead06b241f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349/family/591230670a164f54a3ce4eead06b241f 2024-11-09T16:49:04,134 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349/family/591230670a164f54a3ce4eead06b241f, delay=997 ms 2024-11-09T16:49:04,135 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349/family/591230670a164f54a3ce4eead06b241f, entries=2, sequenceid=6, filesize=4.9 K 2024-11-09T16:49:04,136 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 7228d40e5e10b11f70426f05f75ec349 in 31ms, sequenceid=6, compaction requested=false 2024-11-09T16:49:04,136 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731170943458' 2024-11-09T16:49:04,137 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7228d40e5e10b11f70426f05f75ec349: 2024-11-09T16:49:04,589 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T16:49:05,132 DEBUG [hfile-prefetch-1731170941790 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591230670a164f54a3ce4eead06b241f_0. 2024-11-09T16:49:05,133 DEBUG [hfile-prefetch-1731170941790 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591230670a164f54a3ce4eead06b241f_117. 2024-11-09T16:49:05,133 DEBUG [hfile-prefetch-1731170941790 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349/family/591230670a164f54a3ce4eead06b241f, with total blocks 2, and data blocks 1 2024-11-09T16:49:05,133 DEBUG [hfile-prefetch-1731170941790 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-09T16:49:05,133 DEBUG [hfile-prefetch-1731170941790 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349/family/591230670a164f54a3ce4eead06b241f, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-09T16:49:05,133 DEBUG [hfile-prefetch-1731170941790 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349/family/591230670a164f54a3ce4eead06b241f. Will sleep 100ms and try the verification again. 2024-11-09T16:49:05,137 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-09T16:49:05,137 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-09T16:49:05,137 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;95f00e0c2391:34767,5,FailOnTimeoutGroup] 2024-11-09T16:49:05,137 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-09T16:49:05,137 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-09T16:49:05,137 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '95f00e0c2391,34767,1731170942424' ***** 2024-11-09T16:49:05,137 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-09T16:49:05,137 INFO [RS:1;95f00e0c2391:34767 {}] bucket.BucketCache(1739): Disabling cache 2024-11-09T16:49:05,138 INFO [RS:1;95f00e0c2391:34767 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e/bucket.persistence 2024-11-09T16:49:05,138 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-09T16:49:05,138 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-09T16:49:05,138 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-09T16:49:05,138 INFO [RS:1;95f00e0c2391:34767 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-09T16:49:05,140 ERROR [bucket-cache-persister {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.InterruptedIOException: java.lang.InterruptedException at org.apache.hadoop.util.Shell.runCommand(Shell.java:1071) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1679) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1497) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] Caused by: java.lang.InterruptedException at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at java.lang.ProcessImpl.waitFor(ProcessImpl.java:434) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1061) ~[hadoop-common-3.4.1.jar:?] ... 9 more 2024-11-09T16:49:05,141 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-09T16:49:05,141 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731170945137 2024-11-09T16:49:05,141 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-09T16:49:05,141 INFO [bucket-cache-persister {}] bucket.BucketCachePersister(62): Finishing cache persister thread. 2024-11-09T16:49:05,148 DEBUG [RS:1;95f00e0c2391:34767 {}] bucket.BucketCache(1488): Thread RS:1;95f00e0c2391:34767 started persisting bucket cache to file 2024-11-09T16:49:05,148 DEBUG [RS:1;95f00e0c2391:34767 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-09T16:49:05,148 DEBUG [RS:1;95f00e0c2391:34767 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-09T16:49:05,151 DEBUG [RS:1;95f00e0c2391:34767 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-09T16:49:05,151 DEBUG [RS:1;95f00e0c2391:34767 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731170945148 2024-11-09T16:49:05,151 DEBUG [RS:1;95f00e0c2391:34767 {}] bucket.BucketCache(1510): Thread RS:1;95f00e0c2391:34767 finished persisting bucket cache to file, renaming 2024-11-09T16:49:05,152 INFO [RS:1;95f00e0c2391:34767 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T16:49:05,152 INFO [RS:1;95f00e0c2391:34767 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T16:49:05,152 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T16:49:05,152 INFO [RS:1;95f00e0c2391:34767 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T16:49:05,152 INFO [RS:1;95f00e0c2391:34767 {}] regionserver.HRegionServer(3091): Received CLOSE for 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:05,152 INFO [RS:1;95f00e0c2391:34767 {}] regionserver.HRegionServer(959): stopping server 95f00e0c2391,34767,1731170942424 2024-11-09T16:49:05,152 INFO [RS:1;95f00e0c2391:34767 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T16:49:05,152 INFO [RS:1;95f00e0c2391:34767 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;95f00e0c2391:34767. 2024-11-09T16:49:05,152 INFO [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-09T16:49:05,152 DEBUG [RS:1;95f00e0c2391:34767 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T16:49:05,152 DEBUG [RS:1;95f00e0c2391:34767 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T16:49:05,152 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7228d40e5e10b11f70426f05f75ec349, disabling compactions & flushes 2024-11-09T16:49:05,153 INFO [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. 2024-11-09T16:49:05,153 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. 2024-11-09T16:49:05,153 INFO [RS:1;95f00e0c2391:34767 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-09T16:49:05,153 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. after waiting 0 ms 2024-11-09T16:49:05,153 DEBUG [RS:1;95f00e0c2391:34767 {}] regionserver.HRegionServer(1325): Online Regions={7228d40e5e10b11f70426f05f75ec349=testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349.} 2024-11-09T16:49:05,153 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. 2024-11-09T16:49:05,153 DEBUG [RS:1;95f00e0c2391:34767 {}] regionserver.HRegionServer(1351): Waiting on 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:05,153 DEBUG [StoreFileCloser-7228d40e5e10b11f70426f05f75ec349-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349/family/591230670a164f54a3ce4eead06b241f 2024-11-09T16:49:05,153 WARN [hfile-prefetch-1731170941790 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349/family/591230670a164f54a3ce4eead06b241f, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-09T16:49:05,154 DEBUG [hfile-prefetch-1731170941790 {}] hfile.PrefetchExecutor(121): Prefetch completed for 591230670a164f54a3ce4eead06b241f 2024-11-09T16:49:05,158 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-09T16:49:05,158 INFO [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. 2024-11-09T16:49:05,158 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7228d40e5e10b11f70426f05f75ec349: Waiting for close lock at 1731170945152Running coprocessor pre-close hooks at 1731170945152Disabling compacts and flushes for region at 1731170945152Disabling writes for close at 1731170945153 (+1 ms)Writing region close event to WAL at 1731170945154 (+1 ms)Running coprocessor post-close hooks at 1731170945158 (+4 ms)Closed at 1731170945158 2024-11-09T16:49:05,159 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. 2024-11-09T16:49:05,193 INFO [regionserver/95f00e0c2391:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T16:49:05,353 INFO [RS:1;95f00e0c2391:34767 {}] regionserver.HRegionServer(976): stopping server 95f00e0c2391,34767,1731170942424; all regions closed. 2024-11-09T16:49:05,355 DEBUG [RS:1;95f00e0c2391:34767 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/oldWALs 2024-11-09T16:49:05,355 INFO [RS:1;95f00e0c2391:34767 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 95f00e0c2391%2C34767%2C1731170942424:(num 1731170942931) 2024-11-09T16:49:05,355 DEBUG [RS:1;95f00e0c2391:34767 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T16:49:05,355 INFO [RS:1;95f00e0c2391:34767 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T16:49:05,355 INFO [RS:1;95f00e0c2391:34767 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T16:49:05,355 INFO [RS:1;95f00e0c2391:34767 {}] hbase.ChoreService(370): Chore service for: regionserver/95f00e0c2391:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-09T16:49:05,355 INFO [RS:1;95f00e0c2391:34767 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T16:49:05,355 INFO [RS:1;95f00e0c2391:34767 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T16:49:05,355 INFO [RS:1;95f00e0c2391:34767 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T16:49:05,355 INFO [RS:1;95f00e0c2391:34767 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T16:49:05,355 INFO [regionserver/95f00e0c2391:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T16:49:05,356 INFO [RS:1;95f00e0c2391:34767 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34767 2024-11-09T16:49:05,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34767-0x1018defb2e30002, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/95f00e0c2391,34767,1731170942424 2024-11-09T16:49:05,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T16:49:05,360 INFO [RS:1;95f00e0c2391:34767 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T16:49:05,362 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [95f00e0c2391,34767,1731170942424] 2024-11-09T16:49:05,363 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/95f00e0c2391,34767,1731170942424 already deleted, retry=false 2024-11-09T16:49:05,363 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 95f00e0c2391,34767,1731170942424 on 95f00e0c2391,40569,1731170942346 2024-11-09T16:49:05,368 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 95f00e0c2391,34767,1731170942424, splitWal=true, meta=false 2024-11-09T16:49:05,370 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 95f00e0c2391,34767,1731170942424 (carryingMeta=false) 95f00e0c2391,34767,1731170942424/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@6c0085e0[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-09T16:49:05,370 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 95f00e0c2391,34767,1731170942424, splitWal=true, meta=false 2024-11-09T16:49:05,371 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 95f00e0c2391,34767,1731170942424 had 1 regions 2024-11-09T16:49:05,372 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 95f00e0c2391,34767,1731170942424, splitWal=true, meta=false, isMeta: false 2024-11-09T16:49:05,372 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,34767,1731170942424-splitting 2024-11-09T16:49:05,373 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,34767,1731170942424-splitting dir is empty, no logs to split. 2024-11-09T16:49:05,373 INFO [PEWorker-2 {}] master.SplitWALManager(105): 95f00e0c2391,34767,1731170942424 WAL count=0, meta=false 2024-11-09T16:49:05,374 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,34767,1731170942424-splitting dir is empty, no logs to split. 2024-11-09T16:49:05,374 INFO [PEWorker-2 {}] master.SplitWALManager(105): 95f00e0c2391,34767,1731170942424 WAL count=0, meta=false 2024-11-09T16:49:05,374 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 95f00e0c2391,34767,1731170942424 WAL splitting is done? wals=0, meta=false 2024-11-09T16:49:05,375 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 95f00e0c2391,34767,1731170942424 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,34767,1731170942424-splitting does not exist 2024-11-09T16:49:05,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731170943458, region=7228d40e5e10b11f70426f05f75ec349, ASSIGN}] 2024-11-09T16:49:05,377 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731170943458, region=7228d40e5e10b11f70426f05f75ec349, ASSIGN 2024-11-09T16:49:05,378 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731170943458, region=7228d40e5e10b11f70426f05f75ec349, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T16:49:05,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34767-0x1018defb2e30002, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T16:49:05,462 INFO [RS:1;95f00e0c2391:34767 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T16:49:05,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34767-0x1018defb2e30002, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T16:49:05,462 INFO [RS:1;95f00e0c2391:34767 {}] regionserver.HRegionServer(1031): Exiting; stopping=95f00e0c2391,34767,1731170942424; zookeeper connection closed. 2024-11-09T16:49:05,462 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@726e91bb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@726e91bb 2024-11-09T16:49:05,529 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=7228d40e5e10b11f70426f05f75ec349, regionState=OPENING, regionLocation=95f00e0c2391,37475,1731170942392 2024-11-09T16:49:05,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731170943458, region=7228d40e5e10b11f70426f05f75ec349, ASSIGN because future has completed 2024-11-09T16:49:05,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7228d40e5e10b11f70426f05f75ec349, server=95f00e0c2391,37475,1731170942392}] 2024-11-09T16:49:05,655 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/95f00e0c2391:0 server-side Connection retries=45 2024-11-09T16:49:05,655 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T16:49:05,655 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T16:49:05,655 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T16:49:05,655 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T16:49:05,655 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T16:49:05,655 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T16:49:05,655 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T16:49:05,656 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46173 2024-11-09T16:49:05,658 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46173 connecting to ZooKeeper ensemble=127.0.0.1:51334 2024-11-09T16:49:05,658 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:49:05,658 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:49:05,662 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-09T16:49:05,662 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:461730x0, quorum=127.0.0.1:51334, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-09T16:49:05,663 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T16:49:05,664 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e/bucket.cache 2024-11-09T16:49:05,664 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-09T16:49:05,664 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-09T16:49:05,665 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-09T16:49:05,665 INFO [Thread-202 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-09T16:49:05,665 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-09T16:49:05,665 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T16:49:05,666 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:461730x0, quorum=127.0.0.1:51334, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T16:49:05,666 INFO [Thread-202 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-09T16:49:05,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:461730x0, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T16:49:05,667 INFO [Thread-202 {}] bucket.BucketCache(1706): Retrieved 1 of chunks with blockCount = 1. 2024-11-09T16:49:05,667 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:461730x0, quorum=127.0.0.1:51334, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T16:49:05,668 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46173-0x1018defb2e30003 connected 2024-11-09T16:49:05,668 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46173 2024-11-09T16:49:05,668 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46173 2024-11-09T16:49:05,668 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46173 2024-11-09T16:49:05,669 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46173 2024-11-09T16:49:05,669 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46173 2024-11-09T16:49:05,670 INFO [RS:2;95f00e0c2391:46173 {}] regionserver.HRegionServer(746): ClusterId : 4061dfbb-ef88-4578-93d7-484fbec30f9a 2024-11-09T16:49:05,671 DEBUG [RS:2;95f00e0c2391:46173 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T16:49:05,671 DEBUG [Thread-202 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-09T16:49:05,671 INFO [Thread-202 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-09T16:49:05,671 INFO [Thread-202 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 1 2024-11-09T16:49:05,671 INFO [Thread-202 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e/bucket.persistence is complete. 2024-11-09T16:49:05,673 DEBUG [RS:2;95f00e0c2391:46173 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T16:49:05,673 DEBUG [RS:2;95f00e0c2391:46173 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T16:49:05,675 DEBUG [RS:2;95f00e0c2391:46173 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T16:49:05,676 DEBUG [RS:2;95f00e0c2391:46173 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ca37f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95f00e0c2391/172.17.0.2:0 2024-11-09T16:49:05,686 INFO [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. 2024-11-09T16:49:05,686 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 7228d40e5e10b11f70426f05f75ec349, NAME => 'testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349.', STARTKEY => '', ENDKEY => ''} 2024-11-09T16:49:05,687 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731170943458 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:05,687 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T16:49:05,687 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:05,687 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:05,688 INFO [StoreOpener-7228d40e5e10b11f70426f05f75ec349-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:05,689 INFO [StoreOpener-7228d40e5e10b11f70426f05f75ec349-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7228d40e5e10b11f70426f05f75ec349 columnFamilyName family 2024-11-09T16:49:05,689 DEBUG [RS:2;95f00e0c2391:46173 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;95f00e0c2391:46173 2024-11-09T16:49:05,689 DEBUG [StoreOpener-7228d40e5e10b11f70426f05f75ec349-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T16:49:05,689 INFO [RS:2;95f00e0c2391:46173 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T16:49:05,689 INFO [RS:2;95f00e0c2391:46173 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T16:49:05,689 DEBUG [RS:2;95f00e0c2391:46173 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T16:49:05,690 INFO [RS:2;95f00e0c2391:46173 {}] regionserver.HRegionServer(2659): reportForDuty to master=95f00e0c2391,40569,1731170942346 with port=46173, startcode=1731170945654 2024-11-09T16:49:05,690 DEBUG [RS:2;95f00e0c2391:46173 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T16:49:05,691 DEBUG [StoreFileOpener-7228d40e5e10b11f70426f05f75ec349-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349/family/591230670a164f54a3ce4eead06b241f, delay=972 ms 2024-11-09T16:49:05,692 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38839, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T16:49:05,693 DEBUG [StoreOpener-7228d40e5e10b11f70426f05f75ec349-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349/family/591230670a164f54a3ce4eead06b241f 2024-11-09T16:49:05,693 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40569 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 95f00e0c2391,46173,1731170945654 2024-11-09T16:49:05,693 INFO [StoreOpener-7228d40e5e10b11f70426f05f75ec349-1 {}] regionserver.HStore(327): Store=7228d40e5e10b11f70426f05f75ec349/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T16:49:05,693 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40569 {}] master.ServerManager(517): Registering regionserver=95f00e0c2391,46173,1731170945654 2024-11-09T16:49:05,693 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:05,694 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:05,694 DEBUG [RS:2;95f00e0c2391:46173 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18 2024-11-09T16:49:05,695 DEBUG [RS:2;95f00e0c2391:46173 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-09T16:49:05,695 DEBUG [RS:2;95f00e0c2391:46173 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T16:49:05,695 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:05,695 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:05,695 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:05,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T16:49:05,697 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:05,697 DEBUG [RS:2;95f00e0c2391:46173 {}] zookeeper.ZKUtil(111): regionserver:46173-0x1018defb2e30003, quorum=127.0.0.1:51334, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/95f00e0c2391,46173,1731170945654 2024-11-09T16:49:05,697 WARN [RS:2;95f00e0c2391:46173 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T16:49:05,697 INFO [RS:2;95f00e0c2391:46173 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T16:49:05,697 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [95f00e0c2391,46173,1731170945654] 2024-11-09T16:49:05,697 INFO [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 7228d40e5e10b11f70426f05f75ec349; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63899429, jitterRate=-0.04782430827617645}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T16:49:05,697 DEBUG [RS:2;95f00e0c2391:46173 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,46173,1731170945654 2024-11-09T16:49:05,697 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:05,699 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 7228d40e5e10b11f70426f05f75ec349: Running coprocessor pre-open hook at 1731170945688Writing region info on filesystem at 1731170945688Initializing all the Stores at 1731170945688Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731170945688Cleaning up temporary data from old regions at 1731170945695 (+7 ms)Running coprocessor post-open hooks at 1731170945698 (+3 ms)Region opened successfully at 1731170945699 (+1 ms) 2024-11-09T16:49:05,699 INFO [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349., pid=9, masterSystemTime=1731170945684 2024-11-09T16:49:05,702 DEBUG [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. 2024-11-09T16:49:05,702 INFO [RS_OPEN_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. 2024-11-09T16:49:05,703 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=7228d40e5e10b11f70426f05f75ec349, regionState=OPEN, openSeqNum=10, regionLocation=95f00e0c2391,37475,1731170942392 2024-11-09T16:49:05,703 INFO [RS:2;95f00e0c2391:46173 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T16:49:05,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7228d40e5e10b11f70426f05f75ec349, server=95f00e0c2391,37475,1731170942392 because future has completed 2024-11-09T16:49:05,706 INFO [RS:2;95f00e0c2391:46173 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T16:49:05,707 INFO [RS:2;95f00e0c2391:46173 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T16:49:05,707 INFO [RS:2;95f00e0c2391:46173 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:05,707 INFO [RS:2;95f00e0c2391:46173 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T16:49:05,707 INFO [RS:2;95f00e0c2391:46173 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T16:49:05,708 INFO [RS:2;95f00e0c2391:46173 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:05,708 DEBUG [RS:2;95f00e0c2391:46173 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:05,708 DEBUG [RS:2;95f00e0c2391:46173 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:05,708 DEBUG [RS:2;95f00e0c2391:46173 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:05,708 DEBUG [RS:2;95f00e0c2391:46173 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:05,708 DEBUG [RS:2;95f00e0c2391:46173 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:05,708 DEBUG [RS:2;95f00e0c2391:46173 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/95f00e0c2391:0, corePoolSize=2, maxPoolSize=2 2024-11-09T16:49:05,708 DEBUG [RS:2;95f00e0c2391:46173 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:05,708 DEBUG [RS:2;95f00e0c2391:46173 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:05,708 DEBUG [RS:2;95f00e0c2391:46173 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:05,708 DEBUG [RS:2;95f00e0c2391:46173 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:05,708 DEBUG [RS:2;95f00e0c2391:46173 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:05,709 DEBUG [RS:2;95f00e0c2391:46173 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/95f00e0c2391:0, corePoolSize=1, maxPoolSize=1 2024-11-09T16:49:05,709 DEBUG [RS:2;95f00e0c2391:46173 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/95f00e0c2391:0, corePoolSize=3, maxPoolSize=3 2024-11-09T16:49:05,709 DEBUG [RS:2;95f00e0c2391:46173 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/95f00e0c2391:0, corePoolSize=3, maxPoolSize=3 2024-11-09T16:49:05,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-09T16:49:05,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 7228d40e5e10b11f70426f05f75ec349, server=95f00e0c2391,37475,1731170942392 in 175 msec 2024-11-09T16:49:05,710 INFO [RS:2;95f00e0c2391:46173 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:05,710 INFO [RS:2;95f00e0c2391:46173 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:05,710 INFO [RS:2;95f00e0c2391:46173 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:05,710 INFO [RS:2;95f00e0c2391:46173 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:05,710 INFO [RS:2;95f00e0c2391:46173 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:05,710 INFO [RS:2;95f00e0c2391:46173 {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,46173,1731170945654-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T16:49:05,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-09T16:49:05,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731170943458, region=7228d40e5e10b11f70426f05f75ec349, ASSIGN in 333 msec 2024-11-09T16:49:05,711 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 95f00e0c2391,34767,1731170942424 after splitting done 2024-11-09T16:49:05,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 95f00e0c2391,34767,1731170942424, splitWal=true, meta=false in 347 msec 2024-11-09T16:49:05,726 INFO [RS:2;95f00e0c2391:46173 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T16:49:05,726 INFO [RS:2;95f00e0c2391:46173 {}] hbase.ChoreService(168): Chore ScheduledChore name=95f00e0c2391,46173,1731170945654-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:05,726 INFO [RS:2;95f00e0c2391:46173 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:05,726 INFO [RS:2;95f00e0c2391:46173 {}] regionserver.Replication(171): 95f00e0c2391,46173,1731170945654 started 2024-11-09T16:49:05,741 INFO [RS:2;95f00e0c2391:46173 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T16:49:05,742 INFO [RS:2;95f00e0c2391:46173 {}] regionserver.HRegionServer(1482): Serving as 95f00e0c2391,46173,1731170945654, RpcServer on 95f00e0c2391/172.17.0.2:46173, sessionid=0x1018defb2e30003 2024-11-09T16:49:05,742 DEBUG [RS:2;95f00e0c2391:46173 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T16:49:05,742 DEBUG [RS:2;95f00e0c2391:46173 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 95f00e0c2391,46173,1731170945654 2024-11-09T16:49:05,742 DEBUG [RS:2;95f00e0c2391:46173 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95f00e0c2391,46173,1731170945654' 2024-11-09T16:49:05,742 DEBUG [RS:2;95f00e0c2391:46173 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T16:49:05,743 DEBUG [RS:2;95f00e0c2391:46173 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T16:49:05,743 DEBUG [RS:2;95f00e0c2391:46173 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T16:49:05,743 DEBUG [RS:2;95f00e0c2391:46173 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T16:49:05,743 DEBUG [RS:2;95f00e0c2391:46173 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 95f00e0c2391,46173,1731170945654 2024-11-09T16:49:05,743 DEBUG [RS:2;95f00e0c2391:46173 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95f00e0c2391,46173,1731170945654' 2024-11-09T16:49:05,743 DEBUG [RS:2;95f00e0c2391:46173 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T16:49:05,744 DEBUG [RS:2;95f00e0c2391:46173 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T16:49:05,744 DEBUG [RS:2;95f00e0c2391:46173 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T16:49:05,744 INFO [RS:2;95f00e0c2391:46173 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T16:49:05,744 INFO [RS:2;95f00e0c2391:46173 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T16:49:05,845 INFO [RS:2;95f00e0c2391:46173 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T16:49:05,846 INFO [RS:2;95f00e0c2391:46173 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=95f00e0c2391%2C46173%2C1731170945654, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,46173,1731170945654, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/oldWALs, maxLogs=55 2024-11-09T16:49:05,865 INFO [RS:2;95f00e0c2391:46173 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/WALs/95f00e0c2391,46173,1731170945654/95f00e0c2391%2C46173%2C1731170945654.1731170945847 2024-11-09T16:49:05,866 DEBUG [RS:2;95f00e0c2391:46173 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-09T16:49:06,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-09T16:49:06,242 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T16:49:06,242 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T16:49:06,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T16:49:06,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T16:49:06,242 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T16:49:06,243 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-09T16:49:06,243 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2118365440, stopped=false 2024-11-09T16:49:06,243 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=95f00e0c2391,40569,1731170942346 2024-11-09T16:49:06,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T16:49:06,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46173-0x1018defb2e30003, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T16:49:06,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T16:49:06,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:06,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46173-0x1018defb2e30003, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:06,245 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T16:49:06,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:06,245 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T16:49:06,245 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T16:49:06,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T16:49:06,245 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46173-0x1018defb2e30003, quorum=127.0.0.1:51334, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T16:49:06,245 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T16:49:06,246 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '95f00e0c2391,37475,1731170942392' ***** 2024-11-09T16:49:06,246 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-09T16:49:06,246 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '95f00e0c2391,46173,1731170945654' ***** 2024-11-09T16:49:06,246 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T16:49:06,246 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-09T16:49:06,246 INFO [RS:0;95f00e0c2391:37475 {}] bucket.BucketCache(1739): Disabling cache 2024-11-09T16:49:06,246 INFO [RS:2;95f00e0c2391:46173 {}] bucket.BucketCache(1739): Disabling cache 2024-11-09T16:49:06,246 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-09T16:49:06,246 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-09T16:49:06,246 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-09T16:49:06,246 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-09T16:49:06,247 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-09T16:49:06,247 INFO [RS:0;95f00e0c2391:37475 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e/bucket.persistence 2024-11-09T16:49:06,247 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-09T16:49:06,247 INFO [RS:2;95f00e0c2391:46173 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/835ae2e7-c01e-6b19-cc18-74f25039e42e/bucket.persistence 2024-11-09T16:49:06,247 INFO [RS:2;95f00e0c2391:46173 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-09T16:49:06,247 INFO [RS:0;95f00e0c2391:37475 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-09T16:49:06,247 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-09T16:49:06,247 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-09T16:49:06,257 DEBUG [RS:2;95f00e0c2391:46173 {}] bucket.BucketCache(1488): Thread RS:2;95f00e0c2391:46173 started persisting bucket cache to file 2024-11-09T16:49:06,257 DEBUG [RS:0;95f00e0c2391:37475 {}] bucket.BucketCache(1488): Thread RS:0;95f00e0c2391:37475 started persisting bucket cache to file 2024-11-09T16:49:06,257 DEBUG [RS:2;95f00e0c2391:46173 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-09T16:49:06,257 DEBUG [RS:0;95f00e0c2391:37475 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-09T16:49:06,257 DEBUG [RS:2;95f00e0c2391:46173 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-09T16:49:06,257 DEBUG [RS:0;95f00e0c2391:37475 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-09T16:49:06,261 DEBUG [RS:0;95f00e0c2391:37475 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-09T16:49:06,261 DEBUG [RS:0;95f00e0c2391:37475 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731170946257 2024-11-09T16:49:06,261 DEBUG [RS:2;95f00e0c2391:46173 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-09T16:49:06,261 DEBUG [RS:2;95f00e0c2391:46173 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731170946257 2024-11-09T16:49:06,261 DEBUG [RS:0;95f00e0c2391:37475 {}] bucket.BucketCache(1510): Thread RS:0;95f00e0c2391:37475 finished persisting bucket cache to file, renaming 2024-11-09T16:49:06,261 DEBUG [RS:2;95f00e0c2391:46173 {}] bucket.BucketCache(1510): Thread RS:2;95f00e0c2391:46173 finished persisting bucket cache to file, renaming 2024-11-09T16:49:06,261 WARN [RS:2;95f00e0c2391:46173 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-09T16:49:06,261 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T16:49:06,261 INFO [RS:2;95f00e0c2391:46173 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T16:49:06,261 INFO [RS:0;95f00e0c2391:37475 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T16:49:06,261 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T16:49:06,261 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T16:49:06,261 INFO [RS:2;95f00e0c2391:46173 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T16:49:06,261 INFO [RS:0;95f00e0c2391:37475 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T16:49:06,262 INFO [RS:2;95f00e0c2391:46173 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T16:49:06,262 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.HRegionServer(3091): Received CLOSE for 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:06,262 INFO [RS:2;95f00e0c2391:46173 {}] regionserver.HRegionServer(959): stopping server 95f00e0c2391,46173,1731170945654 2024-11-09T16:49:06,262 INFO [RS:2;95f00e0c2391:46173 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T16:49:06,262 INFO [RS:2;95f00e0c2391:46173 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;95f00e0c2391:46173. 2024-11-09T16:49:06,262 DEBUG [RS:2;95f00e0c2391:46173 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T16:49:06,262 DEBUG [RS:2;95f00e0c2391:46173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T16:49:06,262 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.HRegionServer(959): stopping server 95f00e0c2391,37475,1731170942392 2024-11-09T16:49:06,262 INFO [RS:2;95f00e0c2391:46173 {}] regionserver.HRegionServer(976): stopping server 95f00e0c2391,46173,1731170945654; all regions closed. 2024-11-09T16:49:06,262 INFO [RS:0;95f00e0c2391:37475 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T16:49:06,262 INFO [RS:0;95f00e0c2391:37475 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;95f00e0c2391:37475. 2024-11-09T16:49:06,262 INFO [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-09T16:49:06,262 DEBUG [RS:0;95f00e0c2391:37475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T16:49:06,262 DEBUG [RS:0;95f00e0c2391:37475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T16:49:06,262 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7228d40e5e10b11f70426f05f75ec349, disabling compactions & flushes 2024-11-09T16:49:06,262 INFO [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. 2024-11-09T16:49:06,262 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. 2024-11-09T16:49:06,262 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T16:49:06,262 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. after waiting 0 ms 2024-11-09T16:49:06,262 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T16:49:06,263 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T16:49:06,263 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. 2024-11-09T16:49:06,263 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-09T16:49:06,263 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-09T16:49:06,263 DEBUG [RS:0;95f00e0c2391:37475 {}] regionserver.HRegionServer(1325): Online Regions={7228d40e5e10b11f70426f05f75ec349=testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349., 1588230740=hbase:meta,,1.1588230740} 2024-11-09T16:49:06,263 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-09T16:49:06,263 DEBUG [RS:0;95f00e0c2391:37475 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 7228d40e5e10b11f70426f05f75ec349 2024-11-09T16:49:06,263 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T16:49:06,263 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T16:49:06,263 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T16:49:06,263 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T16:49:06,263 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T16:49:06,264 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-09T16:49:06,264 DEBUG [RS:2;95f00e0c2391:46173 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/oldWALs 2024-11-09T16:49:06,264 INFO [RS:2;95f00e0c2391:46173 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 95f00e0c2391%2C46173%2C1731170945654:(num 1731170945847) 2024-11-09T16:49:06,264 DEBUG [RS:2;95f00e0c2391:46173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T16:49:06,264 INFO [RS:2;95f00e0c2391:46173 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T16:49:06,265 INFO [RS:2;95f00e0c2391:46173 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T16:49:06,265 DEBUG [StoreFileCloser-7228d40e5e10b11f70426f05f75ec349-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349/family/591230670a164f54a3ce4eead06b241f 2024-11-09T16:49:06,265 INFO [RS:2;95f00e0c2391:46173 {}] hbase.ChoreService(370): Chore service for: regionserver/95f00e0c2391:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-09T16:49:06,265 INFO [RS:2;95f00e0c2391:46173 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T16:49:06,265 INFO [regionserver/95f00e0c2391:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T16:49:06,265 INFO [RS:2;95f00e0c2391:46173 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T16:49:06,265 INFO [RS:2;95f00e0c2391:46173 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T16:49:06,265 INFO [RS:2;95f00e0c2391:46173 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T16:49:06,265 INFO [RS:2;95f00e0c2391:46173 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46173 2024-11-09T16:49:06,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46173-0x1018defb2e30003, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/95f00e0c2391,46173,1731170945654 2024-11-09T16:49:06,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T16:49:06,268 INFO [RS:2;95f00e0c2391:46173 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T16:49:06,268 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [95f00e0c2391,46173,1731170945654] 2024-11-09T16:49:06,270 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/95f00e0c2391,46173,1731170945654 already deleted, retry=false 2024-11-09T16:49:06,270 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 95f00e0c2391,46173,1731170945654 expired; onlineServers=1 2024-11-09T16:49:06,270 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/default/testBlockEvictionOnGracefulStop1731170943458/7228d40e5e10b11f70426f05f75ec349/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-09T16:49:06,270 INFO [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. 2024-11-09T16:49:06,270 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7228d40e5e10b11f70426f05f75ec349: Waiting for close lock at 1731170946262Running coprocessor pre-close hooks at 1731170946262Disabling compacts and flushes for region at 1731170946262Disabling writes for close at 1731170946262Writing region close event to WAL at 1731170946265 (+3 ms)Running coprocessor post-close hooks at 1731170946270 (+5 ms)Closed at 1731170946270 2024-11-09T16:49:06,271 DEBUG [RS_CLOSE_REGION-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349. 2024-11-09T16:49:06,276 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ac55a07b29314c92bcf3d404cdbef4d8 2024-11-09T16:49:06,290 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/.tmp/info/ac55a07b29314c92bcf3d404cdbef4d8 is 209, key is testBlockEvictionOnGracefulStop1731170943458,,1731170943462.7228d40e5e10b11f70426f05f75ec349./info:regioninfo/1731170945703/Put/seqid=0 2024-11-09T16:49:06,291 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/.tmp/info/ac55a07b29314c92bcf3d404cdbef4d8 2024-11-09T16:49:06,299 INFO [regionserver/95f00e0c2391:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T16:49:06,299 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0926ea6e870a480a91dd8321cfffe8b8 2024-11-09T16:49:06,310 INFO [regionserver/95f00e0c2391:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T16:49:06,313 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/.tmp/ns/0926ea6e870a480a91dd8321cfffe8b8 is 43, key is default/ns:d/1731170943331/Put/seqid=0 2024-11-09T16:49:06,314 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/.tmp/ns/0926ea6e870a480a91dd8321cfffe8b8 2024-11-09T16:49:06,322 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9591490b9c24469295f9baf99080bcf9 2024-11-09T16:49:06,337 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/.tmp/table/9591490b9c24469295f9baf99080bcf9 is 80, key is testBlockEvictionOnGracefulStop1731170943458/table:state/1731170943833/Put/seqid=0 2024-11-09T16:49:06,338 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/.tmp/table/9591490b9c24469295f9baf99080bcf9 2024-11-09T16:49:06,341 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/.tmp/info/ac55a07b29314c92bcf3d404cdbef4d8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/info/ac55a07b29314c92bcf3d404cdbef4d8 2024-11-09T16:49:06,343 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/info/ac55a07b29314c92bcf3d404cdbef4d8, delay=992 ms 2024-11-09T16:49:06,344 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/info/ac55a07b29314c92bcf3d404cdbef4d8, entries=14, sequenceid=13, filesize=7.5 K 2024-11-09T16:49:06,344 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/.tmp/ns/0926ea6e870a480a91dd8321cfffe8b8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/ns/0926ea6e870a480a91dd8321cfffe8b8 2024-11-09T16:49:06,345 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/ns/0926ea6e870a480a91dd8321cfffe8b8, delay=908 ms 2024-11-09T16:49:06,346 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/ns/0926ea6e870a480a91dd8321cfffe8b8, entries=2, sequenceid=13, filesize=5.0 K 2024-11-09T16:49:06,347 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/.tmp/table/9591490b9c24469295f9baf99080bcf9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/table/9591490b9c24469295f9baf99080bcf9 2024-11-09T16:49:06,348 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/table/9591490b9c24469295f9baf99080bcf9, delay=983 ms 2024-11-09T16:49:06,349 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/table/9591490b9c24469295f9baf99080bcf9, entries=2, sequenceid=13, filesize=5.3 K 2024-11-09T16:49:06,350 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 87ms, sequenceid=13, compaction requested=false 2024-11-09T16:49:06,351 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/info/ac55a07b29314c92bcf3d404cdbef4d8 2024-11-09T16:49:06,351 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/ns/0926ea6e870a480a91dd8321cfffe8b8 2024-11-09T16:49:06,352 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/table/9591490b9c24469295f9baf99080bcf9 2024-11-09T16:49:06,356 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-09T16:49:06,357 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T16:49:06,357 INFO [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T16:49:06,357 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731170946263Running coprocessor pre-close hooks at 1731170946263Disabling compacts and flushes for region at 1731170946263Disabling writes for close at 1731170946263Obtaining lock to block concurrent updates at 1731170946264 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731170946264Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1731170946264Flushing stores of hbase:meta,,1.1588230740 at 1731170946264Flushing 1588230740/info: creating writer at 1731170946264Flushing 1588230740/info: appending metadata at 1731170946290 (+26 ms)Flushing 1588230740/info: closing flushed file at 1731170946290Flushing 1588230740/ns: creating writer at 1731170946293 (+3 ms)Flushing 1588230740/ns: appending metadata at 1731170946313 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731170946313Flushing 1588230740/table: creating writer at 1731170946316 (+3 ms)Flushing 1588230740/table: appending metadata at 1731170946336 (+20 ms)Flushing 1588230740/table: closing flushed file at 1731170946337 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f27199c: reopening flushed file at 1731170946340 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61c2ff36: reopening flushed file at 1731170946344 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26330cdc: reopening flushed file at 1731170946346 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 87ms, sequenceid=13, compaction requested=false at 1731170946350 (+4 ms)Writing region close event to WAL at 1731170946352 (+2 ms)Running coprocessor post-close hooks at 1731170946357 (+5 ms)Closed at 1731170946357 2024-11-09T16:49:06,357 DEBUG [RS_CLOSE_META-regionserver/95f00e0c2391:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-09T16:49:06,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46173-0x1018defb2e30003, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T16:49:06,369 INFO [RS:2;95f00e0c2391:46173 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T16:49:06,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46173-0x1018defb2e30003, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T16:49:06,370 INFO [RS:2;95f00e0c2391:46173 {}] regionserver.HRegionServer(1031): Exiting; stopping=95f00e0c2391,46173,1731170945654; zookeeper connection closed. 2024-11-09T16:49:06,370 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7b717e60 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7b717e60 2024-11-09T16:49:06,463 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.HRegionServer(976): stopping server 95f00e0c2391,37475,1731170942392; all regions closed. 2024-11-09T16:49:06,466 DEBUG [RS:0;95f00e0c2391:37475 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/oldWALs 2024-11-09T16:49:06,466 INFO [RS:0;95f00e0c2391:37475 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 95f00e0c2391%2C37475%2C1731170942392.meta:.meta(num 1731170943275) 2024-11-09T16:49:06,467 DEBUG [RS:0;95f00e0c2391:37475 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/oldWALs 2024-11-09T16:49:06,467 INFO [RS:0;95f00e0c2391:37475 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 95f00e0c2391%2C37475%2C1731170942392:(num 1731170942941) 2024-11-09T16:49:06,467 DEBUG [RS:0;95f00e0c2391:37475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T16:49:06,467 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T16:49:06,468 INFO [RS:0;95f00e0c2391:37475 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T16:49:06,468 INFO [RS:0;95f00e0c2391:37475 {}] hbase.ChoreService(370): Chore service for: regionserver/95f00e0c2391:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-09T16:49:06,468 INFO [RS:0;95f00e0c2391:37475 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T16:49:06,468 INFO [regionserver/95f00e0c2391:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T16:49:06,468 INFO [RS:0;95f00e0c2391:37475 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37475 2024-11-09T16:49:06,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T16:49:06,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/95f00e0c2391,37475,1731170942392 2024-11-09T16:49:06,471 INFO [RS:0;95f00e0c2391:37475 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T16:49:06,472 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [95f00e0c2391,37475,1731170942392] 2024-11-09T16:49:06,474 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/95f00e0c2391,37475,1731170942392 already deleted, retry=false 2024-11-09T16:49:06,474 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 95f00e0c2391,37475,1731170942392 expired; onlineServers=0 2024-11-09T16:49:06,474 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '95f00e0c2391,40569,1731170942346' ***** 2024-11-09T16:49:06,474 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-09T16:49:06,474 INFO [M:0;95f00e0c2391:40569 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T16:49:06,474 INFO [M:0;95f00e0c2391:40569 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T16:49:06,474 DEBUG [M:0;95f00e0c2391:40569 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-09T16:49:06,474 DEBUG [M:0;95f00e0c2391:40569 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-09T16:49:06,474 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-09T16:49:06,474 DEBUG [master/95f00e0c2391:0:becomeActiveMaster-HFileCleaner.large.0-1731170942660 {}] cleaner.HFileCleaner(306): Exit Thread[master/95f00e0c2391:0:becomeActiveMaster-HFileCleaner.large.0-1731170942660,5,FailOnTimeoutGroup] 2024-11-09T16:49:06,474 DEBUG [master/95f00e0c2391:0:becomeActiveMaster-HFileCleaner.small.0-1731170942660 {}] cleaner.HFileCleaner(306): Exit Thread[master/95f00e0c2391:0:becomeActiveMaster-HFileCleaner.small.0-1731170942660,5,FailOnTimeoutGroup] 2024-11-09T16:49:06,475 INFO [M:0;95f00e0c2391:40569 {}] hbase.ChoreService(370): Chore service for: master/95f00e0c2391:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-09T16:49:06,475 INFO [M:0;95f00e0c2391:40569 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T16:49:06,475 DEBUG [M:0;95f00e0c2391:40569 {}] master.HMaster(1795): Stopping service threads 2024-11-09T16:49:06,475 INFO [M:0;95f00e0c2391:40569 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-09T16:49:06,475 INFO [M:0;95f00e0c2391:40569 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T16:49:06,475 INFO [M:0;95f00e0c2391:40569 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T16:49:06,475 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-09T16:49:06,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-09T16:49:06,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T16:49:06,477 DEBUG [M:0;95f00e0c2391:40569 {}] zookeeper.ZKUtil(347): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-09T16:49:06,477 WARN [M:0;95f00e0c2391:40569 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-09T16:49:06,477 INFO [M:0;95f00e0c2391:40569 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/.lastflushedseqids 2024-11-09T16:49:06,484 INFO [M:0;95f00e0c2391:40569 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T16:49:06,484 INFO [M:0;95f00e0c2391:40569 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-09T16:49:06,485 DEBUG [M:0;95f00e0c2391:40569 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T16:49:06,485 INFO [M:0;95f00e0c2391:40569 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T16:49:06,485 DEBUG [M:0;95f00e0c2391:40569 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T16:49:06,485 DEBUG [M:0;95f00e0c2391:40569 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T16:49:06,485 DEBUG [M:0;95f00e0c2391:40569 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T16:49:06,485 INFO [M:0;95f00e0c2391:40569 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-11-09T16:49:06,495 DEBUG [M:0;95f00e0c2391:40569 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2d188e664d334a52a83bf93553b0f00d 2024-11-09T16:49:06,509 DEBUG [M:0;95f00e0c2391:40569 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2d188e664d334a52a83bf93553b0f00d is 82, key is hbase:meta,,1/info:regioninfo/1731170943315/Put/seqid=0 2024-11-09T16:49:06,510 INFO [M:0;95f00e0c2391:40569 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2d188e664d334a52a83bf93553b0f00d 2024-11-09T16:49:06,518 DEBUG [M:0;95f00e0c2391:40569 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6b84eeb09e694cf3b8a49ad8f0a5b0cf 2024-11-09T16:49:06,540 DEBUG [M:0;95f00e0c2391:40569 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6b84eeb09e694cf3b8a49ad8f0a5b0cf is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731170943837/Put/seqid=0 2024-11-09T16:49:06,540 INFO [M:0;95f00e0c2391:40569 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6b84eeb09e694cf3b8a49ad8f0a5b0cf 2024-11-09T16:49:06,548 DEBUG [M:0;95f00e0c2391:40569 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8ca1343069de43da847958d8f8d0c2dc 2024-11-09T16:49:06,563 DEBUG [M:0;95f00e0c2391:40569 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8ca1343069de43da847958d8f8d0c2dc is 69, key is 95f00e0c2391,37475,1731170942392/rs:state/1731170942775/Put/seqid=0 2024-11-09T16:49:06,563 INFO [M:0;95f00e0c2391:40569 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8ca1343069de43da847958d8f8d0c2dc 2024-11-09T16:49:06,565 INFO [M:0;95f00e0c2391:40569 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8ca1343069de43da847958d8f8d0c2dc 2024-11-09T16:49:06,566 DEBUG [M:0;95f00e0c2391:40569 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2d188e664d334a52a83bf93553b0f00d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2d188e664d334a52a83bf93553b0f00d 2024-11-09T16:49:06,568 INFO [M:0;95f00e0c2391:40569 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2d188e664d334a52a83bf93553b0f00d, entries=8, sequenceid=83, filesize=5.5 K 2024-11-09T16:49:06,568 DEBUG [M:0;95f00e0c2391:40569 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6b84eeb09e694cf3b8a49ad8f0a5b0cf as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6b84eeb09e694cf3b8a49ad8f0a5b0cf 2024-11-09T16:49:06,569 INFO [M:0;95f00e0c2391:40569 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6b84eeb09e694cf3b8a49ad8f0a5b0cf, entries=9, sequenceid=83, filesize=6.4 K 2024-11-09T16:49:06,570 DEBUG [M:0;95f00e0c2391:40569 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8ca1343069de43da847958d8f8d0c2dc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8ca1343069de43da847958d8f8d0c2dc 2024-11-09T16:49:06,571 INFO [M:0;95f00e0c2391:40569 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8ca1343069de43da847958d8f8d0c2dc 2024-11-09T16:49:06,571 INFO [M:0;95f00e0c2391:40569 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b5e0631-9a26-7d63-bd22-a69e8e461e1c/6211a6c0-a25e-2ea2-1499-da085c5a5c18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8ca1343069de43da847958d8f8d0c2dc, entries=3, sequenceid=83, filesize=5.3 K 2024-11-09T16:49:06,572 INFO [M:0;95f00e0c2391:40569 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 87ms, sequenceid=83, compaction requested=false 2024-11-09T16:49:06,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T16:49:06,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37475-0x1018defb2e30001, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T16:49:06,573 INFO [RS:0;95f00e0c2391:37475 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T16:49:06,573 INFO [RS:0;95f00e0c2391:37475 {}] regionserver.HRegionServer(1031): Exiting; stopping=95f00e0c2391,37475,1731170942392; zookeeper connection closed. 2024-11-09T16:49:06,573 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3a521450 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3a521450 2024-11-09T16:49:06,573 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-09T16:49:06,574 INFO [M:0;95f00e0c2391:40569 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T16:49:06,574 DEBUG [M:0;95f00e0c2391:40569 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731170946484Disabling compacts and flushes for region at 1731170946484Disabling writes for close at 1731170946485 (+1 ms)Obtaining lock to block concurrent updates at 1731170946485Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731170946485Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1731170946485Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731170946486 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731170946486Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731170946509 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731170946509Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731170946512 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731170946539 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731170946539Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731170946542 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731170946562 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731170946562Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4352e81c: reopening flushed file at 1731170946566 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fa4441c: reopening flushed file at 1731170946568 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70dfd4f3: reopening flushed file at 1731170946570 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 87ms, sequenceid=83, compaction requested=false at 1731170946572 (+2 ms)Writing region close event to WAL at 1731170946574 (+2 ms)Closed at 1731170946574 2024-11-09T16:49:06,575 INFO [M:0;95f00e0c2391:40569 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-09T16:49:06,575 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T16:49:06,575 INFO [M:0;95f00e0c2391:40569 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40569 2024-11-09T16:49:06,576 INFO [M:0;95f00e0c2391:40569 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T16:49:06,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T16:49:06,678 INFO [M:0;95f00e0c2391:40569 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T16:49:06,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40569-0x1018defb2e30000, quorum=127.0.0.1:51334, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T16:49:06,681 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-09T16:49:06,682 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-09T16:49:06,682 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T16:49:06,689 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=187 (was 186) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8643 (was 8719)