2024-12-10 04:44:55,966 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-10 04:44:55,981 main DEBUG Took 0.013421 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-10 04:44:55,981 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-10 04:44:55,981 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-10 04:44:55,982 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-10 04:44:55,983 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 04:44:55,991 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-10 04:44:56,004 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 04:44:56,005 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 04:44:56,006 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 04:44:56,006 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 04:44:56,007 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 04:44:56,007 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 04:44:56,009 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 04:44:56,009 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 04:44:56,010 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 04:44:56,010 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 04:44:56,011 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 04:44:56,012 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 04:44:56,012 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 04:44:56,013 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 04:44:56,013 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 04:44:56,014 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 04:44:56,014 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 04:44:56,014 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 04:44:56,015 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 04:44:56,015 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 04:44:56,016 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 04:44:56,016 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 04:44:56,016 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 04:44:56,017 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 04:44:56,017 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 04:44:56,018 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-10 04:44:56,019 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 04:44:56,021 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-10 04:44:56,023 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-10 04:44:56,023 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-10 04:44:56,025 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-10 04:44:56,026 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-10 04:44:56,035 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-10 04:44:56,038 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-10 04:44:56,040 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-10 04:44:56,040 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-10 04:44:56,041 main DEBUG createAppenders(={Console}) 2024-12-10 04:44:56,042 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-10 04:44:56,042 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-10 04:44:56,042 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-10 04:44:56,043 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-10 04:44:56,043 main DEBUG OutputStream closed 2024-12-10 04:44:56,043 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-10 04:44:56,044 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-10 04:44:56,044 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-10 04:44:56,116 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-10 04:44:56,117 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-10 04:44:56,118 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-10 04:44:56,119 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-10 04:44:56,119 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-10 04:44:56,119 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-10 04:44:56,120 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-10 04:44:56,120 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-10 04:44:56,120 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-10 04:44:56,120 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-10 04:44:56,121 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-10 04:44:56,121 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-10 04:44:56,121 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-10 04:44:56,121 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-10 04:44:56,122 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-10 04:44:56,122 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-10 04:44:56,122 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-10 04:44:56,123 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-10 04:44:56,125 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10 04:44:56,125 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-10 04:44:56,125 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-10 04:44:56,126 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-10T04:44:56,136 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-10 04:44:56,138 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-10 04:44:56,138 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10T04:44:56,369 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2 2024-12-10T04:44:56,391 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=290, ProcessCount=11, AvailableMemoryMB=6091 2024-12-10T04:44:56,479 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-10T04:44:56,554 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:44:56,566 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/cluster_e447825e-eb01-a6b3-6918-2fcb75920c1b, deleteOnExit=true 2024-12-10T04:44:56,673 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/cluster_e447825e-eb01-a6b3-6918-2fcb75920c1b/zookeeper_0, clientPort=59795, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/cluster_e447825e-eb01-a6b3-6918-2fcb75920c1b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/cluster_e447825e-eb01-a6b3-6918-2fcb75920c1b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-10T04:44:56,682 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59795 2024-12-10T04:44:56,683 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:44:56,763 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570 with version=8 2024-12-10T04:44:56,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/hbase-staging 2024-12-10T04:44:56,846 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-10T04:44:57,072 INFO [Time-limited test {}] client.ConnectionUtils(128): master/6578523f4421:0 server-side Connection retries=45 2024-12-10T04:44:57,080 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T04:44:57,081 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T04:44:57,090 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T04:44:57,091 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T04:44:57,091 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T04:44:57,209 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-10T04:44:57,288 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T04:44:57,330 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-10T04:44:57,342 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-10T04:44:57,346 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T04:44:57,381 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 12199 (auto-detected) 2024-12-10T04:44:57,383 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-10T04:44:57,399 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43273 2024-12-10T04:44:57,422 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43273 connecting to ZooKeeper ensemble=127.0.0.1:59795 2024-12-10T04:44:57,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:432730x0, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T04:44:57,533 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43273-0x1000e0a7a970000 connected 2024-12-10T04:44:57,633 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:44:57,634 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:44:57,642 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T04:44:57,646 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570, hbase.cluster.distributed=false 2024-12-10T04:44:57,668 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T04:44:57,675 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43273 2024-12-10T04:44:57,676 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43273 2024-12-10T04:44:57,677 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43273 2024-12-10T04:44:57,678 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43273 2024-12-10T04:44:57,681 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43273 2024-12-10T04:44:57,782 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6578523f4421:0 server-side Connection retries=45 2024-12-10T04:44:57,784 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T04:44:57,785 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T04:44:57,785 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T04:44:57,785 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T04:44:57,786 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T04:44:57,790 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T04:44:57,793 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T04:44:57,795 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43679 2024-12-10T04:44:57,797 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43679 connecting to ZooKeeper ensemble=127.0.0.1:59795 2024-12-10T04:44:57,797 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:44:57,799 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:44:57,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:436790x0, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T04:44:57,821 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:436790x0, quorum=127.0.0.1:59795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T04:44:57,821 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43679-0x1000e0a7a970001 connected 2024-12-10T04:44:57,829 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T04:44:57,848 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/bucket.cache 2024-12-10T04:44:57,857 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T04:44:57,859 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T04:44:57,862 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-10T04:44:57,877 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T04:44:57,877 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-10T04:44:57,893 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T04:44:57,901 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T04:44:57,901 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/bucket.persistence is complete. 2024-12-10T04:44:57,904 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T04:44:57,907 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43679-0x1000e0a7a970001, quorum=127.0.0.1:59795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T04:44:57,915 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43679-0x1000e0a7a970001, quorum=127.0.0.1:59795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T04:44:57,923 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43679 2024-12-10T04:44:57,926 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43679 2024-12-10T04:44:57,934 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43679 2024-12-10T04:44:57,953 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43679 2024-12-10T04:44:57,954 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43679 2024-12-10T04:44:57,984 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6578523f4421:0 server-side Connection retries=45 2024-12-10T04:44:57,984 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T04:44:57,987 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T04:44:57,987 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T04:44:57,988 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T04:44:57,988 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T04:44:57,990 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T04:44:57,992 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T04:44:57,993 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36041 2024-12-10T04:44:57,996 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36041 connecting to ZooKeeper ensemble=127.0.0.1:59795 2024-12-10T04:44:57,997 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:44:57,998 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:44:58,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:360410x0, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T04:44:58,021 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:360410x0, quorum=127.0.0.1:59795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T04:44:58,022 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T04:44:58,025 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36041-0x1000e0a7a970002 connected 2024-12-10T04:44:58,031 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/bucket.cache 2024-12-10T04:44:58,031 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T04:44:58,032 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T04:44:58,032 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-10T04:44:58,032 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T04:44:58,032 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-10T04:44:58,033 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T04:44:58,039 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T04:44:58,039 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/bucket.persistence is complete. 2024-12-10T04:44:58,041 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T04:44:58,043 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36041-0x1000e0a7a970002, quorum=127.0.0.1:59795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T04:44:58,046 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36041-0x1000e0a7a970002, quorum=127.0.0.1:59795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T04:44:58,060 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36041 2024-12-10T04:44:58,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36041 2024-12-10T04:44:58,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36041 2024-12-10T04:44:58,065 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36041 2024-12-10T04:44:58,077 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36041 2024-12-10T04:44:58,103 DEBUG [M:0;6578523f4421:43273 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;6578523f4421:43273 2024-12-10T04:44:58,109 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/6578523f4421,43273,1733805896915 2024-12-10T04:44:58,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36041-0x1000e0a7a970002, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T04:44:58,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T04:44:58,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43679-0x1000e0a7a970001, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T04:44:58,133 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/6578523f4421,43273,1733805896915 2024-12-10T04:44:58,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43679-0x1000e0a7a970001, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T04:44:58,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43679-0x1000e0a7a970001, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:44:58,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36041-0x1000e0a7a970002, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T04:44:58,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36041-0x1000e0a7a970002, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:44:58,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:44:58,174 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T04:44:58,176 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/6578523f4421,43273,1733805896915 from backup master directory 2024-12-10T04:44:58,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36041-0x1000e0a7a970002, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T04:44:58,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/6578523f4421,43273,1733805896915 2024-12-10T04:44:58,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T04:44:58,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43679-0x1000e0a7a970001, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T04:44:58,196 WARN [master/6578523f4421:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T04:44:58,197 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=6578523f4421,43273,1733805896915 2024-12-10T04:44:58,200 INFO [master/6578523f4421:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-10T04:44:58,210 INFO [master/6578523f4421:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-10T04:44:58,221 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/hbase.id] with ID: 6f0f8b6a-0b0d-4820-935b-8fdac6e3fe49 2024-12-10T04:44:58,222 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/.tmp/hbase.id 2024-12-10T04:44:58,229 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/hbase.id] 2024-12-10T04:44:58,357 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:44:58,362 INFO [master/6578523f4421:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-10T04:44:58,386 INFO [master/6578523f4421:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-12-10T04:44:58,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:44:58,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36041-0x1000e0a7a970002, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:44:58,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43679-0x1000e0a7a970001, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:44:58,449 INFO [master/6578523f4421:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T04:44:58,452 INFO [master/6578523f4421:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-10T04:44:58,509 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T04:44:58,517 INFO [master/6578523f4421:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T04:44:58,613 INFO [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store 2024-12-10T04:44:58,638 INFO [master/6578523f4421:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-10T04:44:58,643 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T04:44:58,645 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T04:44:58,646 INFO [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T04:44:58,646 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T04:44:58,649 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T04:44:58,650 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T04:44:58,650 INFO [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T04:44:58,651 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733805898645Disabling compacts and flushes for region at 1733805898645Disabling writes for close at 1733805898649 (+4 ms)Writing region close event to WAL at 1733805898650 (+1 ms)Closed at 1733805898650 2024-12-10T04:44:58,652 WARN [master/6578523f4421:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/.initializing 2024-12-10T04:44:58,652 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/WALs/6578523f4421,43273,1733805896915 2024-12-10T04:44:58,689 INFO [master/6578523f4421:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T04:44:58,719 INFO [master/6578523f4421:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6578523f4421%2C43273%2C1733805896915, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/WALs/6578523f4421,43273,1733805896915, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/oldWALs, maxLogs=10 2024-12-10T04:44:58,777 INFO [master/6578523f4421:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/WALs/6578523f4421,43273,1733805896915/6578523f4421%2C43273%2C1733805896915.1733805898726 2024-12-10T04:44:58,783 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T04:44:58,785 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-10T04:44:58,785 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T04:44:58,790 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:44:58,792 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:44:58,848 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:44:58,888 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-10T04:44:58,894 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:44:58,897 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:44:58,898 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:44:58,903 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-10T04:44:58,904 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:44:58,904 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T04:44:58,905 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:44:58,910 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-10T04:44:58,910 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:44:58,911 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T04:44:58,911 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:44:58,916 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-10T04:44:58,917 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:44:58,917 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T04:44:58,918 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:44:58,923 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:44:58,923 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:44:58,927 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:44:58,928 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:44:58,932 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T04:44:58,935 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:44:58,949 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T04:44:58,950 INFO [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61783864, jitterRate=-0.07934868335723877}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T04:44:58,957 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733805898810Initializing all the Stores at 1733805898811 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733805898811Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733805898814 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733805898815 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733805898815Cleaning up temporary data from old regions at 1733805898928 (+113 ms)Region opened successfully at 1733805898957 (+29 ms) 2024-12-10T04:44:58,958 INFO [master/6578523f4421:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-10T04:44:58,989 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e924572, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6578523f4421/172.17.0.2:0 2024-12-10T04:44:59,024 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-10T04:44:59,038 INFO [master/6578523f4421:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-10T04:44:59,038 INFO [master/6578523f4421:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-10T04:44:59,042 INFO [master/6578523f4421:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-10T04:44:59,044 INFO [master/6578523f4421:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-10T04:44:59,051 INFO [master/6578523f4421:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-10T04:44:59,052 INFO [master/6578523f4421:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-10T04:44:59,090 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-10T04:44:59,106 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-10T04:44:59,117 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-10T04:44:59,120 INFO [master/6578523f4421:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-10T04:44:59,123 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-10T04:44:59,134 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-10T04:44:59,137 INFO [master/6578523f4421:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-10T04:44:59,142 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-10T04:44:59,161 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-10T04:44:59,164 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-10T04:44:59,176 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-10T04:44:59,205 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-10T04:44:59,218 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-10T04:44:59,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43679-0x1000e0a7a970001, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T04:44:59,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36041-0x1000e0a7a970002, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T04:44:59,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T04:44:59,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36041-0x1000e0a7a970002, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:44:59,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43679-0x1000e0a7a970001, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:44:59,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:44:59,283 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=6578523f4421,43273,1733805896915, sessionid=0x1000e0a7a970000, setting cluster-up flag (Was=false) 2024-12-10T04:44:59,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:44:59,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36041-0x1000e0a7a970002, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:44:59,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43679-0x1000e0a7a970001, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:44:59,618 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-10T04:44:59,620 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6578523f4421,43273,1733805896915 2024-12-10T04:44:59,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:44:59,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43679-0x1000e0a7a970001, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:44:59,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36041-0x1000e0a7a970002, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:00,035 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-10T04:45:00,041 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6578523f4421,43273,1733805896915 2024-12-10T04:45:00,047 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-10T04:45:00,083 INFO [RS:0;6578523f4421:43679 {}] regionserver.HRegionServer(746): ClusterId : 6f0f8b6a-0b0d-4820-935b-8fdac6e3fe49 2024-12-10T04:45:00,083 INFO [RS:1;6578523f4421:36041 {}] regionserver.HRegionServer(746): ClusterId : 6f0f8b6a-0b0d-4820-935b-8fdac6e3fe49 2024-12-10T04:45:00,086 DEBUG [RS:1;6578523f4421:36041 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T04:45:00,086 DEBUG [RS:0;6578523f4421:43679 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T04:45:00,115 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-10T04:45:00,124 INFO [master/6578523f4421:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-10T04:45:00,130 INFO [master/6578523f4421:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-10T04:45:00,135 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 6578523f4421,43273,1733805896915 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-10T04:45:00,154 DEBUG [RS:1;6578523f4421:36041 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T04:45:00,154 DEBUG [RS:0;6578523f4421:43679 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T04:45:00,154 DEBUG [RS:1;6578523f4421:36041 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T04:45:00,154 DEBUG [RS:0;6578523f4421:43679 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T04:45:00,154 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/6578523f4421:0, corePoolSize=5, maxPoolSize=5 2024-12-10T04:45:00,155 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/6578523f4421:0, corePoolSize=5, maxPoolSize=5 2024-12-10T04:45:00,155 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/6578523f4421:0, corePoolSize=5, maxPoolSize=5 2024-12-10T04:45:00,155 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/6578523f4421:0, corePoolSize=5, maxPoolSize=5 2024-12-10T04:45:00,155 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/6578523f4421:0, corePoolSize=10, maxPoolSize=10 2024-12-10T04:45:00,156 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,156 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/6578523f4421:0, corePoolSize=2, maxPoolSize=2 2024-12-10T04:45:00,156 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,158 INFO [master/6578523f4421:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733805930158 2024-12-10T04:45:00,160 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-10T04:45:00,161 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-10T04:45:00,162 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T04:45:00,163 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-10T04:45:00,164 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-10T04:45:00,164 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-10T04:45:00,164 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-10T04:45:00,164 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-10T04:45:00,165 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,167 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:00,167 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-10T04:45:00,167 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-10T04:45:00,168 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-10T04:45:00,168 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-10T04:45:00,170 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-10T04:45:00,170 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-10T04:45:00,172 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/6578523f4421:0:becomeActiveMaster-HFileCleaner.large.0-1733805900171,5,FailOnTimeoutGroup] 2024-12-10T04:45:00,173 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/6578523f4421:0:becomeActiveMaster-HFileCleaner.small.0-1733805900172,5,FailOnTimeoutGroup] 2024-12-10T04:45:00,173 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,173 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-10T04:45:00,174 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,175 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,182 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-10T04:45:00,183 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570 2024-12-10T04:45:00,189 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T04:45:00,190 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-10T04:45:00,193 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T04:45:00,193 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:00,194 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:00,194 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-10T04:45:00,197 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T04:45:00,197 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:00,198 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:00,198 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-10T04:45:00,201 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T04:45:00,201 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:00,202 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:00,202 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-10T04:45:00,206 DEBUG [RS:1;6578523f4421:36041 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T04:45:00,206 DEBUG [RS:0;6578523f4421:43679 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T04:45:00,206 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T04:45:00,206 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:00,207 DEBUG [RS:1;6578523f4421:36041 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@417c8caf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6578523f4421/172.17.0.2:0 2024-12-10T04:45:00,207 DEBUG [RS:0;6578523f4421:43679 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@258e1c22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6578523f4421/172.17.0.2:0 2024-12-10T04:45:00,207 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:00,207 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T04:45:00,208 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740 2024-12-10T04:45:00,208 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740 2024-12-10T04:45:00,209 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T04:45:00,209 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T04:45:00,209 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T04:45:00,210 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T04:45:00,217 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T04:45:00,218 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68999957, jitterRate=0.028179481625556946}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T04:45:00,221 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733805900189Initializing all the Stores at 1733805900189Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733805900189Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733805900189Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733805900189Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733805900189Cleaning up temporary data from old regions at 1733805900209 (+20 ms)Region opened successfully at 1733805900221 (+12 ms) 2024-12-10T04:45:00,221 DEBUG [RS:1;6578523f4421:36041 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;6578523f4421:36041 2024-12-10T04:45:00,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T04:45:00,222 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T04:45:00,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T04:45:00,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T04:45:00,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T04:45:00,224 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T04:45:00,224 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733805900222Disabling compacts and flushes for region at 1733805900222Disabling writes for close at 1733805900222Writing region close event to WAL at 1733805900223 (+1 ms)Closed at 1733805900223 2024-12-10T04:45:00,226 INFO [RS:1;6578523f4421:36041 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T04:45:00,226 INFO [RS:1;6578523f4421:36041 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T04:45:00,226 DEBUG [RS:0;6578523f4421:43679 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;6578523f4421:43679 2024-12-10T04:45:00,226 DEBUG [RS:1;6578523f4421:36041 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T04:45:00,227 INFO [RS:0;6578523f4421:43679 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T04:45:00,227 INFO [RS:0;6578523f4421:43679 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T04:45:00,227 DEBUG [RS:0;6578523f4421:43679 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T04:45:00,227 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T04:45:00,227 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-10T04:45:00,230 INFO [RS:1;6578523f4421:36041 {}] regionserver.HRegionServer(2659): reportForDuty to master=6578523f4421,43273,1733805896915 with port=36041, startcode=1733805897982 2024-12-10T04:45:00,230 INFO [RS:0;6578523f4421:43679 {}] regionserver.HRegionServer(2659): reportForDuty to master=6578523f4421,43273,1733805896915 with port=43679, startcode=1733805897723 2024-12-10T04:45:00,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T04:45:00,243 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T04:45:00,245 DEBUG [RS:0;6578523f4421:43679 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T04:45:00,245 DEBUG [RS:1;6578523f4421:36041 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T04:45:00,247 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-10T04:45:00,310 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40861, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T04:45:00,310 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48433, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T04:45:00,318 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43273 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6578523f4421,36041,1733805897982 2024-12-10T04:45:00,320 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43273 {}] master.ServerManager(517): Registering regionserver=6578523f4421,36041,1733805897982 2024-12-10T04:45:00,331 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43273 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6578523f4421,43679,1733805897723 2024-12-10T04:45:00,332 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43273 {}] master.ServerManager(517): Registering regionserver=6578523f4421,43679,1733805897723 2024-12-10T04:45:00,334 DEBUG [RS:0;6578523f4421:43679 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570 2024-12-10T04:45:00,334 DEBUG [RS:1;6578523f4421:36041 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570 2024-12-10T04:45:00,335 DEBUG [RS:0;6578523f4421:43679 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-10T04:45:00,335 DEBUG [RS:1;6578523f4421:36041 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-10T04:45:00,335 DEBUG [RS:0;6578523f4421:43679 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T04:45:00,335 DEBUG [RS:1;6578523f4421:36041 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T04:45:00,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T04:45:00,395 DEBUG [RS:0;6578523f4421:43679 {}] zookeeper.ZKUtil(111): regionserver:43679-0x1000e0a7a970001, quorum=127.0.0.1:59795, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6578523f4421,43679,1733805897723 2024-12-10T04:45:00,395 DEBUG [RS:1;6578523f4421:36041 {}] zookeeper.ZKUtil(111): regionserver:36041-0x1000e0a7a970002, quorum=127.0.0.1:59795, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6578523f4421,36041,1733805897982 2024-12-10T04:45:00,395 WARN [RS:0;6578523f4421:43679 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T04:45:00,395 WARN [RS:1;6578523f4421:36041 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T04:45:00,395 INFO [RS:1;6578523f4421:36041 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T04:45:00,395 INFO [RS:0;6578523f4421:43679 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T04:45:00,396 DEBUG [RS:1;6578523f4421:36041 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,36041,1733805897982 2024-12-10T04:45:00,396 DEBUG [RS:0;6578523f4421:43679 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,43679,1733805897723 2024-12-10T04:45:00,398 WARN [6578523f4421:43273 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-10T04:45:00,398 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6578523f4421,43679,1733805897723] 2024-12-10T04:45:00,399 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6578523f4421,36041,1733805897982] 2024-12-10T04:45:00,421 INFO [RS:1;6578523f4421:36041 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T04:45:00,421 INFO [RS:0;6578523f4421:43679 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T04:45:00,434 INFO [RS:1;6578523f4421:36041 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T04:45:00,434 INFO [RS:0;6578523f4421:43679 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T04:45:00,439 INFO [RS:1;6578523f4421:36041 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T04:45:00,439 INFO [RS:0;6578523f4421:43679 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T04:45:00,440 INFO [RS:1;6578523f4421:36041 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,440 INFO [RS:0;6578523f4421:43679 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,441 INFO [RS:1;6578523f4421:36041 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T04:45:00,441 INFO [RS:0;6578523f4421:43679 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T04:45:00,445 INFO [RS:0;6578523f4421:43679 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T04:45:00,445 INFO [RS:1;6578523f4421:36041 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T04:45:00,447 INFO [RS:1;6578523f4421:36041 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,447 INFO [RS:0;6578523f4421:43679 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,447 DEBUG [RS:0;6578523f4421:43679 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,447 DEBUG [RS:1;6578523f4421:36041 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,447 DEBUG [RS:0;6578523f4421:43679 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,447 DEBUG [RS:1;6578523f4421:36041 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,447 DEBUG [RS:0;6578523f4421:43679 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,447 DEBUG [RS:1;6578523f4421:36041 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,447 DEBUG [RS:0;6578523f4421:43679 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,447 DEBUG [RS:1;6578523f4421:36041 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,448 DEBUG [RS:0;6578523f4421:43679 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,448 DEBUG [RS:1;6578523f4421:36041 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,448 DEBUG [RS:0;6578523f4421:43679 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6578523f4421:0, corePoolSize=2, maxPoolSize=2 2024-12-10T04:45:00,448 DEBUG [RS:1;6578523f4421:36041 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6578523f4421:0, corePoolSize=2, maxPoolSize=2 2024-12-10T04:45:00,448 DEBUG [RS:0;6578523f4421:43679 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,448 DEBUG [RS:1;6578523f4421:36041 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,448 DEBUG [RS:0;6578523f4421:43679 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,448 DEBUG [RS:1;6578523f4421:36041 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,448 DEBUG [RS:0;6578523f4421:43679 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,448 DEBUG [RS:1;6578523f4421:36041 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,448 DEBUG [RS:0;6578523f4421:43679 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,448 DEBUG [RS:1;6578523f4421:36041 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,448 DEBUG [RS:0;6578523f4421:43679 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,448 DEBUG [RS:1;6578523f4421:36041 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,448 DEBUG [RS:0;6578523f4421:43679 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,449 DEBUG [RS:1;6578523f4421:36041 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:00,449 DEBUG [RS:0;6578523f4421:43679 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6578523f4421:0, corePoolSize=3, maxPoolSize=3 2024-12-10T04:45:00,449 DEBUG [RS:1;6578523f4421:36041 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6578523f4421:0, corePoolSize=3, maxPoolSize=3 2024-12-10T04:45:00,449 DEBUG [RS:0;6578523f4421:43679 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6578523f4421:0, corePoolSize=3, maxPoolSize=3 2024-12-10T04:45:00,449 DEBUG [RS:1;6578523f4421:36041 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6578523f4421:0, corePoolSize=3, maxPoolSize=3 2024-12-10T04:45:00,450 INFO [RS:0;6578523f4421:43679 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,450 INFO [RS:0;6578523f4421:43679 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,450 INFO [RS:1;6578523f4421:36041 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,450 INFO [RS:0;6578523f4421:43679 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,450 INFO [RS:1;6578523f4421:36041 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,450 INFO [RS:0;6578523f4421:43679 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,450 INFO [RS:1;6578523f4421:36041 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,450 INFO [RS:0;6578523f4421:43679 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,450 INFO [RS:1;6578523f4421:36041 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,450 INFO [RS:0;6578523f4421:43679 {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,43679,1733805897723-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T04:45:00,450 INFO [RS:1;6578523f4421:36041 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,451 INFO [RS:1;6578523f4421:36041 {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,36041,1733805897982-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T04:45:00,468 INFO [RS:1;6578523f4421:36041 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T04:45:00,470 INFO [RS:1;6578523f4421:36041 {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,36041,1733805897982-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,471 INFO [RS:0;6578523f4421:43679 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T04:45:00,471 INFO [RS:1;6578523f4421:36041 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,471 INFO [RS:1;6578523f4421:36041 {}] regionserver.Replication(171): 6578523f4421,36041,1733805897982 started 2024-12-10T04:45:00,471 INFO [RS:0;6578523f4421:43679 {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,43679,1733805897723-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,471 INFO [RS:0;6578523f4421:43679 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,471 INFO [RS:0;6578523f4421:43679 {}] regionserver.Replication(171): 6578523f4421,43679,1733805897723 started 2024-12-10T04:45:00,486 INFO [RS:1;6578523f4421:36041 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,487 INFO [RS:1;6578523f4421:36041 {}] regionserver.HRegionServer(1482): Serving as 6578523f4421,36041,1733805897982, RpcServer on 6578523f4421/172.17.0.2:36041, sessionid=0x1000e0a7a970002 2024-12-10T04:45:00,488 DEBUG [RS:1;6578523f4421:36041 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T04:45:00,488 DEBUG [RS:1;6578523f4421:36041 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6578523f4421,36041,1733805897982 2024-12-10T04:45:00,488 DEBUG [RS:1;6578523f4421:36041 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6578523f4421,36041,1733805897982' 2024-12-10T04:45:00,489 DEBUG [RS:1;6578523f4421:36041 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T04:45:00,490 DEBUG [RS:1;6578523f4421:36041 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T04:45:00,490 DEBUG [RS:1;6578523f4421:36041 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T04:45:00,490 DEBUG [RS:1;6578523f4421:36041 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T04:45:00,491 DEBUG [RS:1;6578523f4421:36041 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6578523f4421,36041,1733805897982 2024-12-10T04:45:00,491 DEBUG [RS:1;6578523f4421:36041 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6578523f4421,36041,1733805897982' 2024-12-10T04:45:00,491 DEBUG [RS:1;6578523f4421:36041 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T04:45:00,492 INFO [RS:0;6578523f4421:43679 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:00,492 INFO [RS:0;6578523f4421:43679 {}] regionserver.HRegionServer(1482): Serving as 6578523f4421,43679,1733805897723, RpcServer on 6578523f4421/172.17.0.2:43679, sessionid=0x1000e0a7a970001 2024-12-10T04:45:00,492 DEBUG [RS:1;6578523f4421:36041 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T04:45:00,492 DEBUG [RS:0;6578523f4421:43679 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T04:45:00,492 DEBUG [RS:0;6578523f4421:43679 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6578523f4421,43679,1733805897723 2024-12-10T04:45:00,492 DEBUG [RS:0;6578523f4421:43679 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6578523f4421,43679,1733805897723' 2024-12-10T04:45:00,492 DEBUG [RS:0;6578523f4421:43679 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T04:45:00,493 DEBUG [RS:1;6578523f4421:36041 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T04:45:00,493 INFO [RS:1;6578523f4421:36041 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T04:45:00,493 DEBUG [RS:0;6578523f4421:43679 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T04:45:00,493 INFO [RS:1;6578523f4421:36041 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T04:45:00,494 DEBUG [RS:0;6578523f4421:43679 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T04:45:00,494 DEBUG [RS:0;6578523f4421:43679 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T04:45:00,494 DEBUG [RS:0;6578523f4421:43679 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6578523f4421,43679,1733805897723 2024-12-10T04:45:00,494 DEBUG [RS:0;6578523f4421:43679 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6578523f4421,43679,1733805897723' 2024-12-10T04:45:00,494 DEBUG [RS:0;6578523f4421:43679 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T04:45:00,495 DEBUG [RS:0;6578523f4421:43679 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T04:45:00,496 DEBUG [RS:0;6578523f4421:43679 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T04:45:00,496 INFO [RS:0;6578523f4421:43679 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T04:45:00,496 INFO [RS:0;6578523f4421:43679 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T04:45:00,600 INFO [RS:0;6578523f4421:43679 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T04:45:00,600 INFO [RS:1;6578523f4421:36041 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T04:45:00,601 INFO [RS:1;6578523f4421:36041 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6578523f4421%2C36041%2C1733805897982, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,36041,1733805897982, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/oldWALs, maxLogs=55 2024-12-10T04:45:00,601 INFO [RS:0;6578523f4421:43679 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6578523f4421%2C43679%2C1733805897723, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,43679,1733805897723, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/oldWALs, maxLogs=55 2024-12-10T04:45:00,623 INFO [RS:0;6578523f4421:43679 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,43679,1733805897723/6578523f4421%2C43679%2C1733805897723.1733805900605 2024-12-10T04:45:00,623 INFO [RS:1;6578523f4421:36041 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,36041,1733805897982/6578523f4421%2C36041%2C1733805897982.1733805900605 2024-12-10T04:45:00,624 DEBUG [RS:0;6578523f4421:43679 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T04:45:00,624 DEBUG [RS:1;6578523f4421:36041 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T04:45:00,650 DEBUG [6578523f4421:43273 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-10T04:45:00,664 DEBUG [6578523f4421:43273 {}] balancer.BalancerClusterState(204): Hosts are {6578523f4421=0} racks are {/default-rack=0} 2024-12-10T04:45:00,667 DEBUG [6578523f4421:43273 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T04:45:00,667 DEBUG [6578523f4421:43273 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T04:45:00,667 DEBUG [6578523f4421:43273 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T04:45:00,667 DEBUG [6578523f4421:43273 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T04:45:00,667 INFO [6578523f4421:43273 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T04:45:00,667 INFO [6578523f4421:43273 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T04:45:00,668 DEBUG [6578523f4421:43273 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T04:45:00,674 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6578523f4421,36041,1733805897982 2024-12-10T04:45:00,679 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6578523f4421,36041,1733805897982, state=OPENING 2024-12-10T04:45:00,702 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-10T04:45:00,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:00,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36041-0x1000e0a7a970002, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:00,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43679-0x1000e0a7a970001, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:00,713 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T04:45:00,713 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T04:45:00,714 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T04:45:00,715 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T04:45:00,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6578523f4421,36041,1733805897982}] 2024-12-10T04:45:00,886 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T04:45:00,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37393, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T04:45:00,897 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-10T04:45:00,897 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T04:45:00,898 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T04:45:00,900 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6578523f4421%2C36041%2C1733805897982.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,36041,1733805897982, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/oldWALs, maxLogs=55 2024-12-10T04:45:00,927 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,36041,1733805897982/6578523f4421%2C36041%2C1733805897982.meta.1733805900902.meta 2024-12-10T04:45:00,927 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T04:45:00,928 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T04:45:00,930 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T04:45:00,933 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T04:45:00,938 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T04:45:00,942 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T04:45:00,943 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T04:45:00,943 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-10T04:45:00,943 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-10T04:45:00,944 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-10T04:45:00,945 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T04:45:00,946 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:00,946 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:00,946 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-10T04:45:00,947 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T04:45:00,947 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:00,947 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:00,948 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-10T04:45:00,949 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T04:45:00,949 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:00,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:00,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-10T04:45:00,950 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T04:45:00,950 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:00,950 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:00,950 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T04:45:00,951 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740 2024-12-10T04:45:00,952 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740 2024-12-10T04:45:00,952 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T04:45:00,952 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T04:45:00,952 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T04:45:00,954 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T04:45:00,954 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65278273, jitterRate=-0.027277931571006775}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T04:45:00,954 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-10T04:45:00,956 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733805900943Writing region info on filesystem at 1733805900944 (+1 ms)Initializing all the Stores at 1733805900944Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733805900944Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733805900944Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733805900944Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733805900944Cleaning up temporary data from old regions at 1733805900952 (+8 ms)Running coprocessor post-open hooks at 1733805900954 (+2 ms)Region opened successfully at 1733805900955 (+1 ms) 2024-12-10T04:45:00,960 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733805900879 2024-12-10T04:45:00,968 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T04:45:00,969 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-10T04:45:00,970 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=6578523f4421,36041,1733805897982 2024-12-10T04:45:00,971 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6578523f4421,36041,1733805897982, state=OPEN 2024-12-10T04:45:01,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T04:45:01,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36041-0x1000e0a7a970002, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T04:45:01,021 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T04:45:01,021 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6578523f4421,36041,1733805897982 2024-12-10T04:45:01,024 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T04:45:01,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43679-0x1000e0a7a970001, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T04:45:01,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-10T04:45:01,024 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T04:45:01,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6578523f4421,36041,1733805897982 in 305 msec 2024-12-10T04:45:01,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-10T04:45:01,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 791 msec 2024-12-10T04:45:01,031 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T04:45:01,031 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-10T04:45:01,052 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T04:45:01,053 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6578523f4421,36041,1733805897982, seqNum=-1] 2024-12-10T04:45:01,077 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T04:45:01,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42663, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T04:45:01,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0220 sec 2024-12-10T04:45:01,102 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733805901101, completionTime=-1 2024-12-10T04:45:01,105 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-10T04:45:01,105 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-10T04:45:01,133 INFO [master/6578523f4421:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-10T04:45:01,133 INFO [master/6578523f4421:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733805961133 2024-12-10T04:45:01,133 INFO [master/6578523f4421:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733806021133 2024-12-10T04:45:01,133 INFO [master/6578523f4421:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-12-10T04:45:01,136 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-10T04:45:01,143 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,43273,1733805896915-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:01,144 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,43273,1733805896915-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:01,144 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,43273,1733805896915-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:01,146 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-6578523f4421:43273, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:01,146 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:01,147 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:01,152 DEBUG [master/6578523f4421:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-10T04:45:01,179 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.982sec 2024-12-10T04:45:01,180 INFO [master/6578523f4421:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-10T04:45:01,181 INFO [master/6578523f4421:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-10T04:45:01,182 INFO [master/6578523f4421:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-10T04:45:01,183 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-10T04:45:01,183 INFO [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-10T04:45:01,183 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,43273,1733805896915-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T04:45:01,184 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,43273,1733805896915-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-10T04:45:01,185 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-10T04:45:01,186 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-10T04:45:01,187 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,43273,1733805896915-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:01,192 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T04:45:01,194 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-10T04:45:01,194 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-10T04:45:01,197 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6578523f4421,43273,-1 for getting cluster id 2024-12-10T04:45:01,211 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T04:45:01,223 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f0f8b6a-0b0d-4820-935b-8fdac6e3fe49' 2024-12-10T04:45:01,226 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T04:45:01,226 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f0f8b6a-0b0d-4820-935b-8fdac6e3fe49" 2024-12-10T04:45:01,227 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e3cd9cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T04:45:01,227 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6578523f4421,43273,-1] 2024-12-10T04:45:01,229 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T04:45:01,230 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T04:45:01,232 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T04:45:01,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e12cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T04:45:01,236 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T04:45:01,243 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6578523f4421,36041,1733805897982, seqNum=-1] 2024-12-10T04:45:01,244 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T04:45:01,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T04:45:01,266 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=6578523f4421,43273,1733805896915 2024-12-10T04:45:01,270 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T04:45:01,275 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 6578523f4421,43273,1733805896915 2024-12-10T04:45:01,278 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@43e5817c 2024-12-10T04:45:01,279 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T04:45:01,282 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T04:45:01,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43273 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733805901266', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T04:45:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43273 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733805901266 2024-12-10T04:45:01,298 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733805901266 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T04:45:01,300 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43273 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733805901266" procId is: 4 2024-12-10T04:45:01,300 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:01,302 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733805901266 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T04:45:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43273 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T04:45:01,315 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733805901266-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8f745375c4cd8216c22f15e2c50aa2c6, NAME => 'testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733805901266', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570 2024-12-10T04:45:01,322 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733805901266-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T04:45:01,323 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733805901266-pool-0 {}] regionserver.HRegion(1722): Closing 8f745375c4cd8216c22f15e2c50aa2c6, disabling compactions & flushes 2024-12-10T04:45:01,323 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733805901266-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:01,323 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733805901266-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:01,323 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733805901266-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. after waiting 0 ms 2024-12-10T04:45:01,323 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733805901266-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:01,323 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733805901266-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:01,323 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733805901266-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8f745375c4cd8216c22f15e2c50aa2c6: Waiting for close lock at 1733805901323Disabling compacts and flushes for region at 1733805901323Disabling writes for close at 1733805901323Writing region close event to WAL at 1733805901323Closed at 1733805901323 2024-12-10T04:45:01,325 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733805901266 execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T04:45:01,329 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733805901325"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733805901325"}]},"ts":"1733805901325"} 2024-12-10T04:45:01,333 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T04:45:01,334 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733805901266 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T04:45:01,337 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733805901266","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733805901334"}]},"ts":"1733805901334"} 2024-12-10T04:45:01,341 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733805901266, state=ENABLING in hbase:meta 2024-12-10T04:45:01,341 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(204): Hosts are {6578523f4421=0} racks are {/default-rack=0} 2024-12-10T04:45:01,342 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T04:45:01,342 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T04:45:01,342 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T04:45:01,342 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T04:45:01,342 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T04:45:01,342 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T04:45:01,342 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T04:45:01,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733805901266, region=8f745375c4cd8216c22f15e2c50aa2c6, ASSIGN}] 2024-12-10T04:45:01,345 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733805901266, region=8f745375c4cd8216c22f15e2c50aa2c6, ASSIGN 2024-12-10T04:45:01,346 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733805901266, region=8f745375c4cd8216c22f15e2c50aa2c6, ASSIGN; state=OFFLINE, location=6578523f4421,43679,1733805897723; forceNewPlan=false, retain=false 2024-12-10T04:45:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43273 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T04:45:01,500 INFO [6578523f4421:43273 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T04:45:01,502 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8f745375c4cd8216c22f15e2c50aa2c6, regionState=OPENING, regionLocation=6578523f4421,43679,1733805897723 2024-12-10T04:45:01,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733805901266, region=8f745375c4cd8216c22f15e2c50aa2c6, ASSIGN because future has completed 2024-12-10T04:45:01,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f745375c4cd8216c22f15e2c50aa2c6, server=6578523f4421,43679,1733805897723}] 2024-12-10T04:45:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43273 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T04:45:01,662 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T04:45:01,664 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60385, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T04:45:01,667 INFO [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:01,667 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f745375c4cd8216c22f15e2c50aa2c6, NAME => 'testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6.', STARTKEY => '', ENDKEY => ''} 2024-12-10T04:45:01,668 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733805901266 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:01,668 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T04:45:01,668 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:01,669 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:01,670 INFO [StoreOpener-8f745375c4cd8216c22f15e2c50aa2c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:01,674 INFO [StoreOpener-8f745375c4cd8216c22f15e2c50aa2c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f745375c4cd8216c22f15e2c50aa2c6 columnFamilyName family 2024-12-10T04:45:01,675 DEBUG [StoreOpener-8f745375c4cd8216c22f15e2c50aa2c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:01,675 INFO [StoreOpener-8f745375c4cd8216c22f15e2c50aa2c6-1 {}] regionserver.HStore(327): Store=8f745375c4cd8216c22f15e2c50aa2c6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T04:45:01,676 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:01,677 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:01,677 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:01,677 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:01,677 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:01,678 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:01,685 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T04:45:01,686 INFO [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8f745375c4cd8216c22f15e2c50aa2c6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65569845, jitterRate=-0.022933170199394226}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T04:45:01,686 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:01,687 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8f745375c4cd8216c22f15e2c50aa2c6: Running coprocessor pre-open hook at 1733805901669Writing region info on filesystem at 1733805901669Initializing all the Stores at 1733805901669Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733805901669Cleaning up temporary data from old regions at 1733805901677 (+8 ms)Running coprocessor post-open hooks at 1733805901686 (+9 ms)Region opened successfully at 1733805901686 2024-12-10T04:45:01,688 INFO [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6., pid=6, masterSystemTime=1733805901662 2024-12-10T04:45:01,690 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:01,691 INFO [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:01,691 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8f745375c4cd8216c22f15e2c50aa2c6, regionState=OPEN, openSeqNum=2, regionLocation=6578523f4421,43679,1733805897723 2024-12-10T04:45:01,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f745375c4cd8216c22f15e2c50aa2c6, server=6578523f4421,43679,1733805897723 because future has completed 2024-12-10T04:45:01,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-10T04:45:01,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8f745375c4cd8216c22f15e2c50aa2c6, server=6578523f4421,43679,1733805897723 in 187 msec 2024-12-10T04:45:01,700 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-10T04:45:01,700 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733805901266, region=8f745375c4cd8216c22f15e2c50aa2c6, ASSIGN in 355 msec 2024-12-10T04:45:01,700 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733805901266 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T04:45:01,700 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733805901266","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733805901700"}]},"ts":"1733805901700"} 2024-12-10T04:45:01,703 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733805901266, state=ENABLED in hbase:meta 2024-12-10T04:45:01,704 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733805901266 execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T04:45:01,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733805901266 in 412 msec 2024-12-10T04:45:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43273 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T04:45:01,941 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733805901266 get assigned. Timeout = 60000ms 2024-12-10T04:45:01,941 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733805901266 completed 2024-12-10T04:45:01,944 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T04:45:01,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733805901266 assigned to meta. Checking AM states. 2024-12-10T04:45:01,949 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T04:45:01,949 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733805901266 assigned. 2024-12-10T04:45:01,958 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733805901266', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6., hostname=6578523f4421,43679,1733805897723, seqNum=2] 2024-12-10T04:45:01,959 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T04:45:01,962 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T04:45:01,971 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8f745375c4cd8216c22f15e2c50aa2c6 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-10T04:45:01,993 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 543c7a2a67c947faa2c4aa87d3c2201b 2024-12-10T04:45:02,035 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6/.tmp/family/543c7a2a67c947faa2c4aa87d3c2201b is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-10T04:45:02,040 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6/.tmp/family/543c7a2a67c947faa2c4aa87d3c2201b 2024-12-10T04:45:02,070 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6/.tmp/family/543c7a2a67c947faa2c4aa87d3c2201b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6/family/543c7a2a67c947faa2c4aa87d3c2201b 2024-12-10T04:45:02,072 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6/family/543c7a2a67c947faa2c4aa87d3c2201b, delay=946 ms 2024-12-10T04:45:02,075 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6/family/543c7a2a67c947faa2c4aa87d3c2201b, entries=2, sequenceid=6, filesize=4.9 K 2024-12-10T04:45:02,082 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 8f745375c4cd8216c22f15e2c50aa2c6 in 109ms, sequenceid=6, compaction requested=false 2024-12-10T04:45:02,084 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733805901266' 2024-12-10T04:45:02,087 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8f745375c4cd8216c22f15e2c50aa2c6: 2024-12-10T04:45:03,021 DEBUG [hfile-prefetch-1733805902072 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 543c7a2a67c947faa2c4aa87d3c2201b_0. 2024-12-10T04:45:03,025 DEBUG [hfile-prefetch-1733805902072 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 543c7a2a67c947faa2c4aa87d3c2201b_117. 2024-12-10T04:45:03,026 DEBUG [hfile-prefetch-1733805902072 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6/family/543c7a2a67c947faa2c4aa87d3c2201b, with total blocks 2, and data blocks 1 2024-12-10T04:45:03,026 DEBUG [hfile-prefetch-1733805902072 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-10T04:45:03,027 DEBUG [hfile-prefetch-1733805902072 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6/family/543c7a2a67c947faa2c4aa87d3c2201b, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-10T04:45:03,027 DEBUG [hfile-prefetch-1733805902072 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6/family/543c7a2a67c947faa2c4aa87d3c2201b. Will sleep 100ms and try the verification again. 2024-12-10T04:45:03,099 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43273 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=8f745375c4cd8216c22f15e2c50aa2c6, source=6578523f4421,43679,1733805897723, destination=6578523f4421,36041,1733805897982, warming up region on 6578523f4421,36041,1733805897982 2024-12-10T04:45:03,101 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43273 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=8f745375c4cd8216c22f15e2c50aa2c6, source=6578523f4421,43679,1733805897723, destination=6578523f4421,36041,1733805897982, running balancer 2024-12-10T04:45:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43273 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733805901266, region=8f745375c4cd8216c22f15e2c50aa2c6, REOPEN/MOVE 2024-12-10T04:45:03,102 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733805901266, region=8f745375c4cd8216c22f15e2c50aa2c6, REOPEN/MOVE 2024-12-10T04:45:03,102 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36041 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36041 {}] regionserver.HRegion(7855): Warmup {ENCODED => 8f745375c4cd8216c22f15e2c50aa2c6, NAME => 'testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6.', STARTKEY => '', ENDKEY => ''} 2024-12-10T04:45:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36041 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T04:45:03,103 INFO [StoreOpener-8f745375c4cd8216c22f15e2c50aa2c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:03,104 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8f745375c4cd8216c22f15e2c50aa2c6, regionState=CLOSING, regionLocation=6578523f4421,43679,1733805897723 2024-12-10T04:45:03,104 INFO [StoreOpener-8f745375c4cd8216c22f15e2c50aa2c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f745375c4cd8216c22f15e2c50aa2c6 columnFamilyName family 2024-12-10T04:45:03,104 DEBUG [StoreOpener-8f745375c4cd8216c22f15e2c50aa2c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:03,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733805901266, region=8f745375c4cd8216c22f15e2c50aa2c6, REOPEN/MOVE because future has completed 2024-12-10T04:45:03,108 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-10T04:45:03,108 DEBUG [StoreOpener-8f745375c4cd8216c22f15e2c50aa2c6-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6/family/543c7a2a67c947faa2c4aa87d3c2201b 2024-12-10T04:45:03,108 INFO [StoreOpener-8f745375c4cd8216c22f15e2c50aa2c6-1 {}] regionserver.HStore(327): Store=8f745375c4cd8216c22f15e2c50aa2c6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T04:45:03,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8f745375c4cd8216c22f15e2c50aa2c6, server=6578523f4421,43679,1733805897723}] 2024-12-10T04:45:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36041 {}] regionserver.HRegion(1722): Closing 8f745375c4cd8216c22f15e2c50aa2c6, disabling compactions & flushes 2024-12-10T04:45:03,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36041 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36041 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36041 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. after waiting 0 ms 2024-12-10T04:45:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36041 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:03,112 DEBUG [StoreFileCloser-8f745375c4cd8216c22f15e2c50aa2c6-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6/family/543c7a2a67c947faa2c4aa87d3c2201b 2024-12-10T04:45:03,113 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36041 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36041 {}] regionserver.HRegion(1676): Region close journal for 8f745375c4cd8216c22f15e2c50aa2c6: Waiting for close lock at 1733805903108Disabling compacts and flushes for region at 1733805903108Disabling writes for close at 1733805903108Writing region close event to WAL at 1733805903113 (+5 ms)Closed at 1733805903113 2024-12-10T04:45:03,115 WARN [hfile-prefetch-1733805902072 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6/family/543c7a2a67c947faa2c4aa87d3c2201b, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-10T04:45:03,116 DEBUG [hfile-prefetch-1733805902072 {}] hfile.PrefetchExecutor(121): Prefetch completed for 543c7a2a67c947faa2c4aa87d3c2201b 2024-12-10T04:45:03,137 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T04:45:03,137 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T04:45:03,138 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T04:45:03,138 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T04:45:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43273 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-10T04:45:03,156 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T04:45:03,156 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733805903137 2024-12-10T04:45:03,157 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T04:45:03,269 INFO [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:03,269 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-10T04:45:03,270 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 8f745375c4cd8216c22f15e2c50aa2c6, disabling compactions & flushes 2024-12-10T04:45:03,270 INFO [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:03,270 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:03,270 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. after waiting 0 ms 2024-12-10T04:45:03,270 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:03,272 DEBUG [StoreFileCloser-8f745375c4cd8216c22f15e2c50aa2c6-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 543c7a2a67c947faa2c4aa87d3c2201b, starting offset: 0, end offset: 9223372036854775807 2024-12-10T04:45:03,274 DEBUG [StoreFileCloser-8f745375c4cd8216c22f15e2c50aa2c6-family-1 {}] bucket.BucketCache(856): removed key 543c7a2a67c947faa2c4aa87d3c2201b_0 from back map with offset lock 2094080 in the evict process 2024-12-10T04:45:03,279 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-10T04:45:03,280 INFO [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:03,281 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 8f745375c4cd8216c22f15e2c50aa2c6: Waiting for close lock at 1733805903270Running coprocessor pre-close hooks at 1733805903270Disabling compacts and flushes for region at 1733805903270Disabling writes for close at 1733805903270Writing region close event to WAL at 1733805903275 (+5 ms)Running coprocessor post-close hooks at 1733805903279 (+4 ms)Closed at 1733805903280 (+1 ms) 2024-12-10T04:45:03,281 INFO [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 8f745375c4cd8216c22f15e2c50aa2c6 move to 6578523f4421,36041,1733805897982 record at close sequenceid=6 2024-12-10T04:45:03,285 INFO [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:03,285 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8f745375c4cd8216c22f15e2c50aa2c6, regionState=CLOSED 2024-12-10T04:45:03,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8f745375c4cd8216c22f15e2c50aa2c6, server=6578523f4421,43679,1733805897723 because future has completed 2024-12-10T04:45:03,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-10T04:45:03,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 8f745375c4cd8216c22f15e2c50aa2c6, server=6578523f4421,43679,1733805897723 in 181 msec 2024-12-10T04:45:03,292 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733805901266, region=8f745375c4cd8216c22f15e2c50aa2c6, REOPEN/MOVE; state=CLOSED, location=6578523f4421,36041,1733805897982; forceNewPlan=false, retain=false 2024-12-10T04:45:03,357 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T04:45:03,357 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T04:45:03,357 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T04:45:03,357 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T04:45:03,361 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T04:45:03,361 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733805903357 2024-12-10T04:45:03,361 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T04:45:03,442 INFO [6578523f4421:43273 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T04:45:03,443 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8f745375c4cd8216c22f15e2c50aa2c6, regionState=OPENING, regionLocation=6578523f4421,36041,1733805897982 2024-12-10T04:45:03,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733805901266, region=8f745375c4cd8216c22f15e2c50aa2c6, REOPEN/MOVE because future has completed 2024-12-10T04:45:03,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f745375c4cd8216c22f15e2c50aa2c6, server=6578523f4421,36041,1733805897982}] 2024-12-10T04:45:03,603 INFO [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:03,603 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f745375c4cd8216c22f15e2c50aa2c6, NAME => 'testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6.', STARTKEY => '', ENDKEY => ''} 2024-12-10T04:45:03,604 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733805901266 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:03,604 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T04:45:03,604 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:03,604 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:03,605 INFO [StoreOpener-8f745375c4cd8216c22f15e2c50aa2c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:03,606 INFO [StoreOpener-8f745375c4cd8216c22f15e2c50aa2c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f745375c4cd8216c22f15e2c50aa2c6 columnFamilyName family 2024-12-10T04:45:03,606 DEBUG [StoreOpener-8f745375c4cd8216c22f15e2c50aa2c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:03,608 DEBUG [StoreFileOpener-8f745375c4cd8216c22f15e2c50aa2c6-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6/family/543c7a2a67c947faa2c4aa87d3c2201b, delay=969 ms 2024-12-10T04:45:03,610 DEBUG [StoreOpener-8f745375c4cd8216c22f15e2c50aa2c6-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6/family/543c7a2a67c947faa2c4aa87d3c2201b 2024-12-10T04:45:03,610 INFO [StoreOpener-8f745375c4cd8216c22f15e2c50aa2c6-1 {}] regionserver.HStore(327): Store=8f745375c4cd8216c22f15e2c50aa2c6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T04:45:03,611 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:03,611 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:03,612 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:03,612 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:03,612 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:03,616 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:03,617 INFO [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 8f745375c4cd8216c22f15e2c50aa2c6; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59464100, jitterRate=-0.11391586065292358}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T04:45:03,618 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:03,619 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 8f745375c4cd8216c22f15e2c50aa2c6: Running coprocessor pre-open hook at 1733805903604Writing region info on filesystem at 1733805903604Initializing all the Stores at 1733805903605 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733805903605Cleaning up temporary data from old regions at 1733805903612 (+7 ms)Running coprocessor post-open hooks at 1733805903618 (+6 ms)Region opened successfully at 1733805903619 (+1 ms) 2024-12-10T04:45:03,620 INFO [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6., pid=9, masterSystemTime=1733805903600 2024-12-10T04:45:03,622 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:03,623 INFO [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:03,623 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8f745375c4cd8216c22f15e2c50aa2c6, regionState=OPEN, openSeqNum=10, regionLocation=6578523f4421,36041,1733805897982 2024-12-10T04:45:03,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f745375c4cd8216c22f15e2c50aa2c6, server=6578523f4421,36041,1733805897982 because future has completed 2024-12-10T04:45:03,629 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-10T04:45:03,629 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 8f745375c4cd8216c22f15e2c50aa2c6, server=6578523f4421,36041,1733805897982 in 180 msec 2024-12-10T04:45:03,630 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733805901266, region=8f745375c4cd8216c22f15e2c50aa2c6, REOPEN/MOVE in 528 msec 2024-12-10T04:45:03,655 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-10T04:45:03,656 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T04:45:03,656 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T04:45:03,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T04:45:03,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T04:45:03,660 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T04:45:03,660 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-10T04:45:03,660 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-12-10T04:45:03,661 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=6578523f4421,43273,1733805896915 2024-12-10T04:45:03,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43679-0x1000e0a7a970001, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T04:45:03,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36041-0x1000e0a7a970002, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T04:45:03,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T04:45:03,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36041-0x1000e0a7a970002, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:03,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:03,674 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T04:45:03,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43679-0x1000e0a7a970001, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:03,675 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36041-0x1000e0a7a970002, quorum=127.0.0.1:59795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T04:45:03,675 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T04:45:03,675 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T04:45:03,676 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43679-0x1000e0a7a970001, quorum=127.0.0.1:59795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T04:45:03,676 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T04:45:03,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T04:45:03,676 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6578523f4421,43679,1733805897723' ***** 2024-12-10T04:45:03,677 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T04:45:03,677 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6578523f4421,36041,1733805897982' ***** 2024-12-10T04:45:03,677 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T04:45:03,677 INFO [RS:0;6578523f4421:43679 {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T04:45:03,677 INFO [RS:1;6578523f4421:36041 {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T04:45:03,677 INFO [RS:1;6578523f4421:36041 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/bucket.persistence 2024-12-10T04:45:03,677 INFO [RS:0;6578523f4421:43679 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/bucket.persistence 2024-12-10T04:45:03,677 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T04:45:03,677 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T04:45:03,677 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T04:45:03,677 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T04:45:03,677 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T04:45:03,678 INFO [RS:1;6578523f4421:36041 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T04:45:03,678 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T04:45:03,678 INFO [RS:0;6578523f4421:43679 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T04:45:03,678 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T04:45:03,678 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T04:45:03,688 DEBUG [RS:1;6578523f4421:36041 {}] bucket.BucketCache(1487): Thread RS:1;6578523f4421:36041 started persisting bucket cache to file 2024-12-10T04:45:03,688 DEBUG [RS:0;6578523f4421:43679 {}] bucket.BucketCache(1487): Thread RS:0;6578523f4421:43679 started persisting bucket cache to file 2024-12-10T04:45:03,688 DEBUG [RS:1;6578523f4421:36041 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T04:45:03,688 DEBUG [RS:1;6578523f4421:36041 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T04:45:03,688 DEBUG [RS:0;6578523f4421:43679 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T04:45:03,688 DEBUG [RS:0;6578523f4421:43679 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T04:45:03,692 DEBUG [RS:0;6578523f4421:43679 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T04:45:03,692 DEBUG [RS:1;6578523f4421:36041 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T04:45:03,692 DEBUG [RS:0;6578523f4421:43679 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733805903688 2024-12-10T04:45:03,692 DEBUG [RS:1;6578523f4421:36041 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733805903688 2024-12-10T04:45:03,692 DEBUG [RS:1;6578523f4421:36041 {}] bucket.BucketCache(1509): Thread RS:1;6578523f4421:36041 finished persisting bucket cache to file, renaming 2024-12-10T04:45:03,692 DEBUG [RS:0;6578523f4421:43679 {}] bucket.BucketCache(1509): Thread RS:0;6578523f4421:43679 finished persisting bucket cache to file, renaming 2024-12-10T04:45:03,692 WARN [RS:0;6578523f4421:43679 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-10T04:45:03,692 INFO [RS:1;6578523f4421:36041 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T04:45:03,692 INFO [RS:0;6578523f4421:43679 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T04:45:03,692 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T04:45:03,692 INFO [RS:1;6578523f4421:36041 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T04:45:03,692 INFO [RS:0;6578523f4421:43679 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T04:45:03,693 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T04:45:03,693 INFO [RS:0;6578523f4421:43679 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T04:45:03,693 INFO [RS:1;6578523f4421:36041 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T04:45:03,693 INFO [RS:0;6578523f4421:43679 {}] regionserver.HRegionServer(959): stopping server 6578523f4421,43679,1733805897723 2024-12-10T04:45:03,693 INFO [RS:1;6578523f4421:36041 {}] regionserver.HRegionServer(3091): Received CLOSE for 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:03,693 INFO [RS:0;6578523f4421:43679 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T04:45:03,693 INFO [RS:0;6578523f4421:43679 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;6578523f4421:43679. 2024-12-10T04:45:03,693 DEBUG [RS:0;6578523f4421:43679 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T04:45:03,693 DEBUG [RS:0;6578523f4421:43679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T04:45:03,693 INFO [RS:0;6578523f4421:43679 {}] regionserver.HRegionServer(976): stopping server 6578523f4421,43679,1733805897723; all regions closed. 2024-12-10T04:45:03,694 INFO [RS:1;6578523f4421:36041 {}] regionserver.HRegionServer(959): stopping server 6578523f4421,36041,1733805897982 2024-12-10T04:45:03,694 INFO [RS:1;6578523f4421:36041 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T04:45:03,694 INFO [RS:1;6578523f4421:36041 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;6578523f4421:36041. 2024-12-10T04:45:03,694 DEBUG [RS:1;6578523f4421:36041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T04:45:03,694 INFO [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T04:45:03,694 DEBUG [RS:1;6578523f4421:36041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T04:45:03,695 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8f745375c4cd8216c22f15e2c50aa2c6, disabling compactions & flushes 2024-12-10T04:45:03,695 INFO [RS:1;6578523f4421:36041 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T04:45:03,695 INFO [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:03,695 INFO [RS:1;6578523f4421:36041 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T04:45:03,695 INFO [RS:1;6578523f4421:36041 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T04:45:03,695 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:03,695 INFO [RS:1;6578523f4421:36041 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-10T04:45:03,695 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. after waiting 0 ms 2024-12-10T04:45:03,695 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:03,696 INFO [RS:1;6578523f4421:36041 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-10T04:45:03,696 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T04:45:03,696 DEBUG [StoreFileCloser-8f745375c4cd8216c22f15e2c50aa2c6-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6/family/543c7a2a67c947faa2c4aa87d3c2201b 2024-12-10T04:45:03,696 DEBUG [RS:1;6578523f4421:36041 {}] regionserver.HRegionServer(1325): Online Regions={8f745375c4cd8216c22f15e2c50aa2c6=testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6., 1588230740=hbase:meta,,1.1588230740} 2024-12-10T04:45:03,696 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T04:45:03,696 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T04:45:03,696 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T04:45:03,696 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T04:45:03,696 DEBUG [RS:1;6578523f4421:36041 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 8f745375c4cd8216c22f15e2c50aa2c6 2024-12-10T04:45:03,696 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T04:45:03,696 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-10T04:45:03,699 DEBUG [RS:0;6578523f4421:43679 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/oldWALs 2024-12-10T04:45:03,699 INFO [RS:0;6578523f4421:43679 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6578523f4421%2C43679%2C1733805897723:(num 1733805900605) 2024-12-10T04:45:03,699 DEBUG [RS:0;6578523f4421:43679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T04:45:03,699 INFO [RS:0;6578523f4421:43679 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T04:45:03,700 INFO [RS:0;6578523f4421:43679 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T04:45:03,700 INFO [RS:0;6578523f4421:43679 {}] hbase.ChoreService(370): Chore service for: regionserver/6578523f4421:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T04:45:03,700 INFO [RS:0;6578523f4421:43679 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T04:45:03,700 INFO [RS:0;6578523f4421:43679 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T04:45:03,700 INFO [regionserver/6578523f4421:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T04:45:03,700 INFO [RS:0;6578523f4421:43679 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T04:45:03,700 INFO [RS:0;6578523f4421:43679 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T04:45:03,701 INFO [RS:0;6578523f4421:43679 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43679 2024-12-10T04:45:03,707 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9ba6065be52d4a109dfae51231aa125c 2024-12-10T04:45:03,707 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnRegionMove1733805901266/8f745375c4cd8216c22f15e2c50aa2c6/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-10T04:45:03,707 INFO [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:03,707 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8f745375c4cd8216c22f15e2c50aa2c6: Waiting for close lock at 1733805903694Running coprocessor pre-close hooks at 1733805903694Disabling compacts and flushes for region at 1733805903695 (+1 ms)Disabling writes for close at 1733805903695Writing region close event to WAL at 1733805903696 (+1 ms)Running coprocessor post-close hooks at 1733805903707 (+11 ms)Closed at 1733805903707 2024-12-10T04:45:03,707 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6. 2024-12-10T04:45:03,723 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/info/9ba6065be52d4a109dfae51231aa125c is 205, key is testBlockEvictionOnRegionMove1733805901266,,1733805901283.8f745375c4cd8216c22f15e2c50aa2c6./info:regioninfo/1733805903623/Put/seqid=0 2024-12-10T04:45:03,725 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/info/9ba6065be52d4a109dfae51231aa125c 2024-12-10T04:45:03,734 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0850c1b130fc4cd29e2fc855d72692a3 2024-12-10T04:45:03,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43679-0x1000e0a7a970001, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6578523f4421,43679,1733805897723 2024-12-10T04:45:03,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T04:45:03,737 INFO [RS:0;6578523f4421:43679 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T04:45:03,747 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/ns/0850c1b130fc4cd29e2fc855d72692a3 is 43, key is default/ns:d/1733805901084/Put/seqid=0 2024-12-10T04:45:03,748 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/ns/0850c1b130fc4cd29e2fc855d72692a3 2024-12-10T04:45:03,753 INFO [regionserver/6578523f4421:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T04:45:03,753 INFO [regionserver/6578523f4421:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T04:45:03,759 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6578523f4421,43679,1733805897723] 2024-12-10T04:45:03,767 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fdc4069d12974526aac021092f8edb41 2024-12-10T04:45:03,778 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/table/fdc4069d12974526aac021092f8edb41 is 78, key is testBlockEvictionOnRegionMove1733805901266/table:state/1733805901700/Put/seqid=0 2024-12-10T04:45:03,779 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/table/fdc4069d12974526aac021092f8edb41 2024-12-10T04:45:03,782 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/info/9ba6065be52d4a109dfae51231aa125c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/info/9ba6065be52d4a109dfae51231aa125c 2024-12-10T04:45:03,783 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/info/9ba6065be52d4a109dfae51231aa125c, delay=929 ms 2024-12-10T04:45:03,784 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/info/9ba6065be52d4a109dfae51231aa125c, entries=15, sequenceid=15, filesize=7.6 K 2024-12-10T04:45:03,784 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/ns/0850c1b130fc4cd29e2fc855d72692a3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/ns/0850c1b130fc4cd29e2fc855d72692a3 2024-12-10T04:45:03,785 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/ns/0850c1b130fc4cd29e2fc855d72692a3, delay=980 ms 2024-12-10T04:45:03,787 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/ns/0850c1b130fc4cd29e2fc855d72692a3, entries=2, sequenceid=15, filesize=5.0 K 2024-12-10T04:45:03,788 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/table/fdc4069d12974526aac021092f8edb41 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/table/fdc4069d12974526aac021092f8edb41 2024-12-10T04:45:03,789 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/table/fdc4069d12974526aac021092f8edb41, delay=911 ms 2024-12-10T04:45:03,790 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/table/fdc4069d12974526aac021092f8edb41, entries=2, sequenceid=15, filesize=5.3 K 2024-12-10T04:45:03,791 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 95ms, sequenceid=15, compaction requested=false 2024-12-10T04:45:03,791 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-10T04:45:03,793 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/info/9ba6065be52d4a109dfae51231aa125c 2024-12-10T04:45:03,795 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/ns/0850c1b130fc4cd29e2fc855d72692a3 2024-12-10T04:45:03,795 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/table/fdc4069d12974526aac021092f8edb41 2024-12-10T04:45:03,799 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-10T04:45:03,800 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T04:45:03,800 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T04:45:03,800 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733805903696Running coprocessor pre-close hooks at 1733805903696Disabling compacts and flushes for region at 1733805903696Disabling writes for close at 1733805903696Obtaining lock to block concurrent updates at 1733805903696Preparing flush snapshotting stores in 1588230740 at 1733805903696Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733805903697 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733805903697Flushing 1588230740/info: creating writer at 1733805903698 (+1 ms)Flushing 1588230740/info: appending metadata at 1733805903722 (+24 ms)Flushing 1588230740/info: closing flushed file at 1733805903722Flushing 1588230740/ns: creating writer at 1733805903728 (+6 ms)Flushing 1588230740/ns: appending metadata at 1733805903746 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1733805903746Flushing 1588230740/table: creating writer at 1733805903751 (+5 ms)Flushing 1588230740/table: appending metadata at 1733805903778 (+27 ms)Flushing 1588230740/table: closing flushed file at 1733805903778Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74ae803b: reopening flushed file at 1733805903781 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4402f98: reopening flushed file at 1733805903784 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9b9a54b: reopening flushed file at 1733805903787 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 95ms, sequenceid=15, compaction requested=false at 1733805903791 (+4 ms)Writing region close event to WAL at 1733805903795 (+4 ms)Running coprocessor post-close hooks at 1733805903800 (+5 ms)Closed at 1733805903800 2024-12-10T04:45:03,801 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T04:45:03,820 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6578523f4421,43679,1733805897723 already deleted, retry=false 2024-12-10T04:45:03,821 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6578523f4421,43679,1733805897723 expired; onlineServers=1 2024-12-10T04:45:03,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43679-0x1000e0a7a970001, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T04:45:03,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43679-0x1000e0a7a970001, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T04:45:03,859 INFO [RS:0;6578523f4421:43679 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T04:45:03,860 INFO [RS:0;6578523f4421:43679 {}] regionserver.HRegionServer(1031): Exiting; stopping=6578523f4421,43679,1733805897723; zookeeper connection closed. 2024-12-10T04:45:03,860 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7cec66dd {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7cec66dd 2024-12-10T04:45:03,896 INFO [RS:1;6578523f4421:36041 {}] regionserver.HRegionServer(976): stopping server 6578523f4421,36041,1733805897982; all regions closed. 2024-12-10T04:45:03,899 DEBUG [RS:1;6578523f4421:36041 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/oldWALs 2024-12-10T04:45:03,899 INFO [RS:1;6578523f4421:36041 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6578523f4421%2C36041%2C1733805897982.meta:.meta(num 1733805900902) 2024-12-10T04:45:03,901 DEBUG [RS:1;6578523f4421:36041 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/oldWALs 2024-12-10T04:45:03,901 INFO [RS:1;6578523f4421:36041 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6578523f4421%2C36041%2C1733805897982:(num 1733805900605) 2024-12-10T04:45:03,901 DEBUG [RS:1;6578523f4421:36041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T04:45:03,901 INFO [RS:1;6578523f4421:36041 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T04:45:03,902 INFO [RS:1;6578523f4421:36041 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T04:45:03,903 INFO [RS:1;6578523f4421:36041 {}] hbase.ChoreService(370): Chore service for: regionserver/6578523f4421:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T04:45:03,903 INFO [RS:1;6578523f4421:36041 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T04:45:03,903 INFO [regionserver/6578523f4421:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T04:45:03,903 INFO [RS:1;6578523f4421:36041 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36041 2024-12-10T04:45:03,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T04:45:03,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36041-0x1000e0a7a970002, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6578523f4421,36041,1733805897982 2024-12-10T04:45:03,916 INFO [RS:1;6578523f4421:36041 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T04:45:03,927 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6578523f4421,36041,1733805897982] 2024-12-10T04:45:03,937 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6578523f4421,36041,1733805897982 already deleted, retry=false 2024-12-10T04:45:03,937 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6578523f4421,36041,1733805897982 expired; onlineServers=0 2024-12-10T04:45:03,938 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '6578523f4421,43273,1733805896915' ***** 2024-12-10T04:45:03,938 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-10T04:45:03,938 INFO [M:0;6578523f4421:43273 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T04:45:03,938 INFO [M:0;6578523f4421:43273 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T04:45:03,938 DEBUG [M:0;6578523f4421:43273 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-10T04:45:03,938 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-10T04:45:03,938 DEBUG [M:0;6578523f4421:43273 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-10T04:45:03,938 DEBUG [master/6578523f4421:0:becomeActiveMaster-HFileCleaner.large.0-1733805900171 {}] cleaner.HFileCleaner(306): Exit Thread[master/6578523f4421:0:becomeActiveMaster-HFileCleaner.large.0-1733805900171,5,FailOnTimeoutGroup] 2024-12-10T04:45:03,938 DEBUG [master/6578523f4421:0:becomeActiveMaster-HFileCleaner.small.0-1733805900172 {}] cleaner.HFileCleaner(306): Exit Thread[master/6578523f4421:0:becomeActiveMaster-HFileCleaner.small.0-1733805900172,5,FailOnTimeoutGroup] 2024-12-10T04:45:03,939 INFO [M:0;6578523f4421:43273 {}] hbase.ChoreService(370): Chore service for: master/6578523f4421:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-10T04:45:03,939 INFO [M:0;6578523f4421:43273 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T04:45:03,939 DEBUG [M:0;6578523f4421:43273 {}] master.HMaster(1795): Stopping service threads 2024-12-10T04:45:03,939 INFO [M:0;6578523f4421:43273 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-10T04:45:03,939 INFO [M:0;6578523f4421:43273 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T04:45:03,939 INFO [M:0;6578523f4421:43273 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-10T04:45:03,940 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-10T04:45:03,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-10T04:45:03,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:03,948 DEBUG [M:0;6578523f4421:43273 {}] zookeeper.ZKUtil(347): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-10T04:45:03,948 WARN [M:0;6578523f4421:43273 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-10T04:45:03,948 INFO [M:0;6578523f4421:43273 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/.lastflushedseqids 2024-12-10T04:45:03,958 INFO [M:0;6578523f4421:43273 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-10T04:45:03,958 INFO [M:0;6578523f4421:43273 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-10T04:45:03,958 DEBUG [M:0;6578523f4421:43273 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T04:45:03,958 INFO [M:0;6578523f4421:43273 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T04:45:03,958 DEBUG [M:0;6578523f4421:43273 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T04:45:03,959 DEBUG [M:0;6578523f4421:43273 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T04:45:03,959 DEBUG [M:0;6578523f4421:43273 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T04:45:03,959 INFO [M:0;6578523f4421:43273 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.66 KB heapSize=42.99 KB 2024-12-10T04:45:03,967 DEBUG [M:0;6578523f4421:43273 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 023d0221e33445388be96f999b8f26f4 2024-12-10T04:45:03,980 DEBUG [M:0;6578523f4421:43273 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/023d0221e33445388be96f999b8f26f4 is 82, key is hbase:meta,,1/info:regioninfo/1733805900969/Put/seqid=0 2024-12-10T04:45:03,981 INFO [M:0;6578523f4421:43273 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/023d0221e33445388be96f999b8f26f4 2024-12-10T04:45:03,990 DEBUG [M:0;6578523f4421:43273 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 52a050ed59cb4d978e63cdaea83fed5d 2024-12-10T04:45:04,004 DEBUG [M:0;6578523f4421:43273 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/52a050ed59cb4d978e63cdaea83fed5d is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733805901705/Put/seqid=0 2024-12-10T04:45:04,005 INFO [M:0;6578523f4421:43273 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.04 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/52a050ed59cb4d978e63cdaea83fed5d 2024-12-10T04:45:04,022 DEBUG [M:0;6578523f4421:43273 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ce7530709f084e78a08c60253ba75208 2024-12-10T04:45:04,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36041-0x1000e0a7a970002, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T04:45:04,027 INFO [RS:1;6578523f4421:36041 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T04:45:04,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36041-0x1000e0a7a970002, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T04:45:04,027 INFO [RS:1;6578523f4421:36041 {}] regionserver.HRegionServer(1031): Exiting; stopping=6578523f4421,36041,1733805897982; zookeeper connection closed. 2024-12-10T04:45:04,029 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1a0946c3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1a0946c3 2024-12-10T04:45:04,029 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-10T04:45:04,035 DEBUG [M:0;6578523f4421:43273 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ce7530709f084e78a08c60253ba75208 is 69, key is 6578523f4421,36041,1733805897982/rs:state/1733805900323/Put/seqid=0 2024-12-10T04:45:04,036 INFO [M:0;6578523f4421:43273 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ce7530709f084e78a08c60253ba75208 2024-12-10T04:45:04,038 DEBUG [M:0;6578523f4421:43273 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/023d0221e33445388be96f999b8f26f4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/023d0221e33445388be96f999b8f26f4 2024-12-10T04:45:04,046 INFO [M:0;6578523f4421:43273 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/023d0221e33445388be96f999b8f26f4, entries=8, sequenceid=84, filesize=5.5 K 2024-12-10T04:45:04,047 DEBUG [M:0;6578523f4421:43273 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/52a050ed59cb4d978e63cdaea83fed5d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/52a050ed59cb4d978e63cdaea83fed5d 2024-12-10T04:45:04,051 INFO [M:0;6578523f4421:43273 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/52a050ed59cb4d978e63cdaea83fed5d, entries=9, sequenceid=84, filesize=6.5 K 2024-12-10T04:45:04,052 DEBUG [M:0;6578523f4421:43273 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ce7530709f084e78a08c60253ba75208 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ce7530709f084e78a08c60253ba75208 2024-12-10T04:45:04,055 INFO [M:0;6578523f4421:43273 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ce7530709f084e78a08c60253ba75208, entries=2, sequenceid=84, filesize=5.1 K 2024-12-10T04:45:04,056 INFO [M:0;6578523f4421:43273 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 97ms, sequenceid=84, compaction requested=false 2024-12-10T04:45:04,063 INFO [M:0;6578523f4421:43273 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T04:45:04,064 DEBUG [M:0;6578523f4421:43273 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733805903958Disabling compacts and flushes for region at 1733805903958Disabling writes for close at 1733805903959 (+1 ms)Obtaining lock to block concurrent updates at 1733805903959Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733805903959Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35494, getHeapSize=43960, getOffHeapSize=0, getCellsCount=99 at 1733805903959Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733805903960 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733805903960Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733805903980 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733805903980Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733805903984 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733805904003 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733805904003Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733805904007 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733805904035 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733805904035Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d2ee87e: reopening flushed file at 1733805904037 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a80c17b: reopening flushed file at 1733805904047 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c24cb7d: reopening flushed file at 1733805904052 (+5 ms)Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 97ms, sequenceid=84, compaction requested=false at 1733805904056 (+4 ms)Writing region close event to WAL at 1733805904063 (+7 ms)Closed at 1733805904063 2024-12-10T04:45:04,067 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T04:45:04,068 INFO [M:0;6578523f4421:43273 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-10T04:45:04,068 INFO [M:0;6578523f4421:43273 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43273 2024-12-10T04:45:04,068 INFO [M:0;6578523f4421:43273 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T04:45:04,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T04:45:04,181 INFO [M:0;6578523f4421:43273 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T04:45:04,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43273-0x1000e0a7a970000, quorum=127.0.0.1:59795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T04:45:04,189 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-10T04:45:04,208 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-10T04:45:04,208 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:45:04,216 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733805903608 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1733805902072 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1733805903785 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/6578523f4421:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/6578523f4421:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/6578523f4421:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733805903783 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: regionserver/6578523f4421:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=300 (was 290) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5647 (was 6091) 2024-12-10T04:45:04,222 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=300, ProcessCount=11, AvailableMemoryMB=5647 2024-12-10T04:45:04,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/hadoop-log-dir so I do NOT create it in target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630 2024-12-10T04:45:04,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/hadoop-tmp-dir so I do NOT create it in target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630 2024-12-10T04:45:04,223 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:45:04,229 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630/cluster_7e34e5c0-70c8-bb28-ecb6-20bc5b340f04, deleteOnExit=true 2024-12-10T04:45:04,233 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630/cluster_7e34e5c0-70c8-bb28-ecb6-20bc5b340f04/zookeeper_0, clientPort=57720, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630/cluster_7e34e5c0-70c8-bb28-ecb6-20bc5b340f04/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630/cluster_7e34e5c0-70c8-bb28-ecb6-20bc5b340f04/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-10T04:45:04,234 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57720 2024-12-10T04:45:04,245 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570 with version=8 2024-12-10T04:45:04,245 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/hbase-staging 2024-12-10T04:45:04,249 INFO [Time-limited test {}] client.ConnectionUtils(128): master/6578523f4421:0 server-side Connection retries=45 2024-12-10T04:45:04,249 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T04:45:04,249 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T04:45:04,250 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T04:45:04,250 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T04:45:04,250 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T04:45:04,250 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-10T04:45:04,250 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T04:45:04,253 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42569 2024-12-10T04:45:04,255 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42569 connecting to ZooKeeper ensemble=127.0.0.1:57720 2024-12-10T04:45:04,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:425690x0, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T04:45:04,322 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42569-0x1000e0a982f0000 connected 2024-12-10T04:45:04,402 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:45:04,403 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:45:04,404 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T04:45:04,404 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570, hbase.cluster.distributed=false 2024-12-10T04:45:04,406 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T04:45:04,407 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42569 2024-12-10T04:45:04,407 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42569 2024-12-10T04:45:04,407 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42569 2024-12-10T04:45:04,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42569 2024-12-10T04:45:04,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42569 2024-12-10T04:45:04,421 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6578523f4421:0 server-side Connection retries=45 2024-12-10T04:45:04,421 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T04:45:04,421 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T04:45:04,421 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T04:45:04,421 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T04:45:04,421 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T04:45:04,421 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T04:45:04,422 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T04:45:04,422 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42295 2024-12-10T04:45:04,425 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42295 connecting to ZooKeeper ensemble=127.0.0.1:57720 2024-12-10T04:45:04,425 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:45:04,425 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:45:04,434 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:422950x0, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T04:45:04,435 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:422950x0, quorum=127.0.0.1:57720, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T04:45:04,435 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T04:45:04,437 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42295-0x1000e0a982f0001 connected 2024-12-10T04:45:04,437 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630/bucket.cache 2024-12-10T04:45:04,437 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T04:45:04,437 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T04:45:04,438 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-10T04:45:04,438 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T04:45:04,438 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-10T04:45:04,438 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T04:45:04,439 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T04:45:04,439 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T04:45:04,439 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630/bucket.persistence is complete. 2024-12-10T04:45:04,440 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T04:45:04,441 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T04:45:04,442 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42295 2024-12-10T04:45:04,442 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42295 2024-12-10T04:45:04,443 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42295 2024-12-10T04:45:04,443 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42295 2024-12-10T04:45:04,443 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42295 2024-12-10T04:45:04,462 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6578523f4421:0 server-side Connection retries=45 2024-12-10T04:45:04,462 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T04:45:04,463 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T04:45:04,463 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T04:45:04,463 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T04:45:04,463 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T04:45:04,463 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T04:45:04,463 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T04:45:04,465 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46207 2024-12-10T04:45:04,467 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46207 connecting to ZooKeeper ensemble=127.0.0.1:57720 2024-12-10T04:45:04,467 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:45:04,468 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:45:04,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:462070x0, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T04:45:04,487 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:462070x0, quorum=127.0.0.1:57720, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T04:45:04,487 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46207-0x1000e0a982f0002 connected 2024-12-10T04:45:04,487 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T04:45:04,488 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630/bucket.cache 2024-12-10T04:45:04,488 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T04:45:04,488 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T04:45:04,488 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-10T04:45:04,489 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T04:45:04,489 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T04:45:04,489 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-10T04:45:04,489 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T04:45:04,489 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630/bucket.persistence is complete. 2024-12-10T04:45:04,490 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T04:45:04,491 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46207-0x1000e0a982f0002, quorum=127.0.0.1:57720, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T04:45:04,492 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46207-0x1000e0a982f0002, quorum=127.0.0.1:57720, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T04:45:04,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46207 2024-12-10T04:45:04,497 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46207 2024-12-10T04:45:04,497 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46207 2024-12-10T04:45:04,497 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46207 2024-12-10T04:45:04,499 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46207 2024-12-10T04:45:04,510 DEBUG [M:0;6578523f4421:42569 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;6578523f4421:42569 2024-12-10T04:45:04,511 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/6578523f4421,42569,1733805904248 2024-12-10T04:45:04,518 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T04:45:04,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46207-0x1000e0a982f0002, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T04:45:04,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T04:45:04,519 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/6578523f4421,42569,1733805904248 2024-12-10T04:45:04,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46207-0x1000e0a982f0002, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T04:45:04,528 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T04:45:04,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:04,529 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:04,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46207-0x1000e0a982f0002, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:04,529 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T04:45:04,530 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/6578523f4421,42569,1733805904248 from backup master directory 2024-12-10T04:45:04,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/6578523f4421,42569,1733805904248 2024-12-10T04:45:04,539 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T04:45:04,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46207-0x1000e0a982f0002, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T04:45:04,539 WARN [master/6578523f4421:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T04:45:04,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T04:45:04,539 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=6578523f4421,42569,1733805904248 2024-12-10T04:45:04,540 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/hbase.id] with ID: 98590c83-ab1a-4d90-943c-331e4788e621 2024-12-10T04:45:04,540 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/.tmp/hbase.id 2024-12-10T04:45:04,543 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/hbase.id] 2024-12-10T04:45:04,573 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:45:04,573 INFO [master/6578523f4421:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-10T04:45:04,576 INFO [master/6578523f4421:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-12-10T04:45:04,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46207-0x1000e0a982f0002, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:04,590 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:04,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:04,594 INFO [master/6578523f4421:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T04:45:04,595 INFO [master/6578523f4421:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-10T04:45:04,595 INFO [master/6578523f4421:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T04:45:04,614 INFO [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store 2024-12-10T04:45:04,620 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T04:45:04,620 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T04:45:04,621 INFO [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T04:45:04,621 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T04:45:04,621 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T04:45:04,621 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T04:45:04,621 INFO [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T04:45:04,621 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733805904620Disabling compacts and flushes for region at 1733805904620Disabling writes for close at 1733805904621 (+1 ms)Writing region close event to WAL at 1733805904621Closed at 1733805904621 2024-12-10T04:45:04,621 WARN [master/6578523f4421:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/.initializing 2024-12-10T04:45:04,621 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/WALs/6578523f4421,42569,1733805904248 2024-12-10T04:45:04,626 INFO [master/6578523f4421:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T04:45:04,630 INFO [master/6578523f4421:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6578523f4421%2C42569%2C1733805904248, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/WALs/6578523f4421,42569,1733805904248, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/oldWALs, maxLogs=10 2024-12-10T04:45:04,653 INFO [master/6578523f4421:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/WALs/6578523f4421,42569,1733805904248/6578523f4421%2C42569%2C1733805904248.1733805904631 2024-12-10T04:45:04,653 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T04:45:04,653 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-10T04:45:04,654 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T04:45:04,654 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:45:04,654 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:45:04,655 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:45:04,661 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-10T04:45:04,662 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:04,662 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:04,662 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:45:04,666 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-10T04:45:04,667 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:04,667 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T04:45:04,667 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:45:04,671 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-10T04:45:04,671 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:04,672 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T04:45:04,672 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:45:04,676 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-10T04:45:04,676 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:04,676 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T04:45:04,677 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:45:04,677 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:45:04,677 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:45:04,678 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:45:04,678 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:45:04,678 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T04:45:04,678 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T04:45:04,684 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T04:45:04,685 INFO [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74617295, jitterRate=0.11188434064388275}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T04:45:04,686 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733805904654Initializing all the Stores at 1733805904654Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733805904655 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733805904655Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733805904655Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733805904655Cleaning up temporary data from old regions at 1733805904678 (+23 ms)Region opened successfully at 1733805904686 (+8 ms) 2024-12-10T04:45:04,687 INFO [master/6578523f4421:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-10T04:45:04,690 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aaf66d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6578523f4421/172.17.0.2:0 2024-12-10T04:45:04,691 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-10T04:45:04,691 INFO [master/6578523f4421:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-10T04:45:04,691 INFO [master/6578523f4421:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-10T04:45:04,691 INFO [master/6578523f4421:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-10T04:45:04,692 INFO [master/6578523f4421:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-10T04:45:04,692 INFO [master/6578523f4421:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-10T04:45:04,692 INFO [master/6578523f4421:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-10T04:45:04,694 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-10T04:45:04,695 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-10T04:45:04,707 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-10T04:45:04,708 INFO [master/6578523f4421:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-10T04:45:04,708 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-10T04:45:04,718 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-10T04:45:04,718 INFO [master/6578523f4421:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-10T04:45:04,720 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-10T04:45:04,728 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-10T04:45:04,730 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-10T04:45:04,739 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-10T04:45:04,742 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-10T04:45:04,749 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-10T04:45:04,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46207-0x1000e0a982f0002, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T04:45:04,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T04:45:04,760 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T04:45:04,760 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:04,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:04,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46207-0x1000e0a982f0002, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:04,761 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=6578523f4421,42569,1733805904248, sessionid=0x1000e0a982f0000, setting cluster-up flag (Was=false) 2024-12-10T04:45:04,781 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:04,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:04,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46207-0x1000e0a982f0002, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:04,813 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-10T04:45:04,815 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6578523f4421,42569,1733805904248 2024-12-10T04:45:04,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:04,834 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:04,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46207-0x1000e0a982f0002, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:04,876 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-10T04:45:04,878 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6578523f4421,42569,1733805904248 2024-12-10T04:45:04,878 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-10T04:45:04,880 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-10T04:45:04,880 INFO [master/6578523f4421:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-10T04:45:04,881 INFO [master/6578523f4421:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-10T04:45:04,881 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 6578523f4421,42569,1733805904248 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-10T04:45:04,883 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/6578523f4421:0, corePoolSize=5, maxPoolSize=5 2024-12-10T04:45:04,884 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/6578523f4421:0, corePoolSize=5, maxPoolSize=5 2024-12-10T04:45:04,884 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/6578523f4421:0, corePoolSize=5, maxPoolSize=5 2024-12-10T04:45:04,884 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/6578523f4421:0, corePoolSize=5, maxPoolSize=5 2024-12-10T04:45:04,884 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/6578523f4421:0, corePoolSize=10, maxPoolSize=10 2024-12-10T04:45:04,884 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:04,884 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/6578523f4421:0, corePoolSize=2, maxPoolSize=2 2024-12-10T04:45:04,884 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:04,885 INFO [master/6578523f4421:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733805934885 2024-12-10T04:45:04,886 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-10T04:45:04,886 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-10T04:45:04,886 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-10T04:45:04,886 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-10T04:45:04,886 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-10T04:45:04,886 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-10T04:45:04,886 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T04:45:04,886 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-10T04:45:04,887 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:04,887 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:04,887 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-10T04:45:04,887 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-10T04:45:04,887 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-10T04:45:04,887 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-10T04:45:04,888 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-10T04:45:04,888 INFO [master/6578523f4421:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-10T04:45:04,888 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/6578523f4421:0:becomeActiveMaster-HFileCleaner.large.0-1733805904888,5,FailOnTimeoutGroup] 2024-12-10T04:45:04,888 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/6578523f4421:0:becomeActiveMaster-HFileCleaner.small.0-1733805904888,5,FailOnTimeoutGroup] 2024-12-10T04:45:04,888 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:04,889 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-10T04:45:04,889 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:04,889 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:04,901 INFO [RS:0;6578523f4421:42295 {}] regionserver.HRegionServer(746): ClusterId : 98590c83-ab1a-4d90-943c-331e4788e621 2024-12-10T04:45:04,901 INFO [RS:1;6578523f4421:46207 {}] regionserver.HRegionServer(746): ClusterId : 98590c83-ab1a-4d90-943c-331e4788e621 2024-12-10T04:45:04,901 DEBUG [RS:1;6578523f4421:46207 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T04:45:04,901 DEBUG [RS:0;6578523f4421:42295 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T04:45:04,903 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-10T04:45:04,903 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570 2024-12-10T04:45:04,908 DEBUG [RS:1;6578523f4421:46207 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T04:45:04,908 DEBUG [RS:0;6578523f4421:42295 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T04:45:04,908 DEBUG [RS:0;6578523f4421:42295 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T04:45:04,908 DEBUG [RS:1;6578523f4421:46207 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T04:45:04,913 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T04:45:04,913 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-10T04:45:04,916 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T04:45:04,916 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:04,916 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:04,916 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-10T04:45:04,919 DEBUG [RS:1;6578523f4421:46207 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T04:45:04,919 DEBUG [RS:0;6578523f4421:42295 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T04:45:04,919 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T04:45:04,919 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:04,920 DEBUG [RS:0;6578523f4421:42295 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4221cdb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6578523f4421/172.17.0.2:0 2024-12-10T04:45:04,920 DEBUG [RS:1;6578523f4421:46207 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24c5501e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6578523f4421/172.17.0.2:0 2024-12-10T04:45:04,920 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:04,920 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-10T04:45:04,930 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T04:45:04,930 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:04,930 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:04,930 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-10T04:45:04,932 DEBUG [RS:1;6578523f4421:46207 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;6578523f4421:46207 2024-12-10T04:45:04,932 INFO [RS:1;6578523f4421:46207 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T04:45:04,932 INFO [RS:1;6578523f4421:46207 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T04:45:04,932 DEBUG [RS:1;6578523f4421:46207 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T04:45:04,933 INFO [RS:1;6578523f4421:46207 {}] regionserver.HRegionServer(2659): reportForDuty to master=6578523f4421,42569,1733805904248 with port=46207, startcode=1733805904462 2024-12-10T04:45:04,933 DEBUG [RS:1;6578523f4421:46207 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T04:45:04,933 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T04:45:04,933 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:04,933 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:04,934 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T04:45:04,934 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740 2024-12-10T04:45:04,934 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740 2024-12-10T04:45:04,934 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T04:45:04,935 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T04:45:04,935 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T04:45:04,935 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37859, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T04:45:04,935 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T04:45:04,936 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42569 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6578523f4421,46207,1733805904462 2024-12-10T04:45:04,936 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42569 {}] master.ServerManager(517): Registering regionserver=6578523f4421,46207,1733805904462 2024-12-10T04:45:04,936 DEBUG [RS:0;6578523f4421:42295 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;6578523f4421:42295 2024-12-10T04:45:04,936 INFO [RS:0;6578523f4421:42295 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T04:45:04,936 INFO [RS:0;6578523f4421:42295 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T04:45:04,936 DEBUG [RS:0;6578523f4421:42295 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T04:45:04,937 INFO [RS:0;6578523f4421:42295 {}] regionserver.HRegionServer(2659): reportForDuty to master=6578523f4421,42569,1733805904248 with port=42295, startcode=1733805904421 2024-12-10T04:45:04,938 DEBUG [RS:0;6578523f4421:42295 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T04:45:04,938 DEBUG [RS:1;6578523f4421:46207 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570 2024-12-10T04:45:04,938 DEBUG [RS:1;6578523f4421:46207 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-10T04:45:04,938 DEBUG [RS:1;6578523f4421:46207 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T04:45:04,940 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37115, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T04:45:04,940 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42569 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6578523f4421,42295,1733805904421 2024-12-10T04:45:04,940 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42569 {}] master.ServerManager(517): Registering regionserver=6578523f4421,42295,1733805904421 2024-12-10T04:45:04,941 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T04:45:04,942 DEBUG [RS:0;6578523f4421:42295 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570 2024-12-10T04:45:04,942 DEBUG [RS:0;6578523f4421:42295 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-10T04:45:04,942 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60742040, jitterRate=-0.09487307071685791}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T04:45:04,942 DEBUG [RS:0;6578523f4421:42295 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T04:45:04,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733805904913Initializing all the Stores at 1733805904913Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733805904913Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733805904913Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733805904913Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733805904913Cleaning up temporary data from old regions at 1733805904935 (+22 ms)Region opened successfully at 1733805904942 (+7 ms) 2024-12-10T04:45:04,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T04:45:04,942 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T04:45:04,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T04:45:04,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T04:45:04,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T04:45:04,943 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T04:45:04,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733805904942Disabling compacts and flushes for region at 1733805904942Disabling writes for close at 1733805904942Writing region close event to WAL at 1733805904943 (+1 ms)Closed at 1733805904943 2024-12-10T04:45:04,944 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T04:45:04,944 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-10T04:45:04,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T04:45:04,945 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T04:45:04,945 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-10T04:45:04,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T04:45:04,997 DEBUG [RS:1;6578523f4421:46207 {}] zookeeper.ZKUtil(111): regionserver:46207-0x1000e0a982f0002, quorum=127.0.0.1:57720, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6578523f4421,46207,1733805904462 2024-12-10T04:45:04,997 WARN [RS:1;6578523f4421:46207 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T04:45:04,997 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6578523f4421,46207,1733805904462] 2024-12-10T04:45:04,997 INFO [RS:1;6578523f4421:46207 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T04:45:04,997 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6578523f4421,42295,1733805904421] 2024-12-10T04:45:04,998 DEBUG [RS:1;6578523f4421:46207 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,46207,1733805904462 2024-12-10T04:45:04,998 DEBUG [RS:0;6578523f4421:42295 {}] zookeeper.ZKUtil(111): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6578523f4421,42295,1733805904421 2024-12-10T04:45:04,998 WARN [RS:0;6578523f4421:42295 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T04:45:04,998 INFO [RS:0;6578523f4421:42295 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T04:45:04,998 DEBUG [RS:0;6578523f4421:42295 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,42295,1733805904421 2024-12-10T04:45:05,006 INFO [RS:1;6578523f4421:46207 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T04:45:05,006 INFO [RS:0;6578523f4421:42295 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T04:45:05,008 INFO [RS:1;6578523f4421:46207 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T04:45:05,012 INFO [RS:1;6578523f4421:46207 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T04:45:05,012 INFO [RS:0;6578523f4421:42295 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T04:45:05,012 INFO [RS:1;6578523f4421:46207 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,015 INFO [RS:1;6578523f4421:46207 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T04:45:05,015 INFO [RS:0;6578523f4421:42295 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T04:45:05,015 INFO [RS:0;6578523f4421:42295 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,015 INFO [RS:0;6578523f4421:42295 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T04:45:05,015 INFO [RS:1;6578523f4421:46207 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T04:45:05,016 INFO [RS:1;6578523f4421:46207 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,016 DEBUG [RS:1;6578523f4421:46207 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,016 INFO [RS:0;6578523f4421:42295 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T04:45:05,016 DEBUG [RS:1;6578523f4421:46207 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,016 INFO [RS:0;6578523f4421:42295 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,016 DEBUG [RS:1;6578523f4421:46207 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,016 DEBUG [RS:0;6578523f4421:42295 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,016 DEBUG [RS:1;6578523f4421:46207 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,016 DEBUG [RS:0;6578523f4421:42295 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,016 DEBUG [RS:1;6578523f4421:46207 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,016 DEBUG [RS:0;6578523f4421:42295 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,016 DEBUG [RS:1;6578523f4421:46207 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6578523f4421:0, corePoolSize=2, maxPoolSize=2 2024-12-10T04:45:05,017 DEBUG [RS:0;6578523f4421:42295 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,017 DEBUG [RS:1;6578523f4421:46207 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,017 DEBUG [RS:1;6578523f4421:46207 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,017 DEBUG [RS:0;6578523f4421:42295 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,017 DEBUG [RS:1;6578523f4421:46207 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,017 DEBUG [RS:0;6578523f4421:42295 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6578523f4421:0, corePoolSize=2, maxPoolSize=2 2024-12-10T04:45:05,017 DEBUG [RS:0;6578523f4421:42295 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,017 DEBUG [RS:1;6578523f4421:46207 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,017 DEBUG [RS:0;6578523f4421:42295 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,017 DEBUG [RS:0;6578523f4421:42295 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,017 DEBUG [RS:1;6578523f4421:46207 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,017 DEBUG [RS:0;6578523f4421:42295 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,017 DEBUG [RS:1;6578523f4421:46207 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,017 DEBUG [RS:0;6578523f4421:42295 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,017 DEBUG [RS:0;6578523f4421:42295 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:05,017 DEBUG [RS:1;6578523f4421:46207 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6578523f4421:0, corePoolSize=3, maxPoolSize=3 2024-12-10T04:45:05,018 DEBUG [RS:0;6578523f4421:42295 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6578523f4421:0, corePoolSize=3, maxPoolSize=3 2024-12-10T04:45:05,018 DEBUG [RS:1;6578523f4421:46207 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6578523f4421:0, corePoolSize=3, maxPoolSize=3 2024-12-10T04:45:05,018 DEBUG [RS:0;6578523f4421:42295 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6578523f4421:0, corePoolSize=3, maxPoolSize=3 2024-12-10T04:45:05,019 INFO [RS:0;6578523f4421:42295 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,019 INFO [RS:1;6578523f4421:46207 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,019 INFO [RS:0;6578523f4421:42295 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,019 INFO [RS:1;6578523f4421:46207 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,019 INFO [RS:0;6578523f4421:42295 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,019 INFO [RS:1;6578523f4421:46207 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,019 INFO [RS:0;6578523f4421:42295 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,019 INFO [RS:1;6578523f4421:46207 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,019 INFO [RS:0;6578523f4421:42295 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,019 INFO [RS:1;6578523f4421:46207 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,019 INFO [RS:0;6578523f4421:42295 {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,42295,1733805904421-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T04:45:05,019 INFO [RS:1;6578523f4421:46207 {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,46207,1733805904462-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T04:45:05,035 INFO [RS:1;6578523f4421:46207 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T04:45:05,035 INFO [RS:0;6578523f4421:42295 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T04:45:05,035 INFO [RS:0;6578523f4421:42295 {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,42295,1733805904421-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,035 INFO [RS:1;6578523f4421:46207 {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,46207,1733805904462-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,035 INFO [RS:0;6578523f4421:42295 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,035 INFO [RS:1;6578523f4421:46207 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,035 INFO [RS:0;6578523f4421:42295 {}] regionserver.Replication(171): 6578523f4421,42295,1733805904421 started 2024-12-10T04:45:05,035 INFO [RS:1;6578523f4421:46207 {}] regionserver.Replication(171): 6578523f4421,46207,1733805904462 started 2024-12-10T04:45:05,048 INFO [RS:0;6578523f4421:42295 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,048 INFO [RS:0;6578523f4421:42295 {}] regionserver.HRegionServer(1482): Serving as 6578523f4421,42295,1733805904421, RpcServer on 6578523f4421/172.17.0.2:42295, sessionid=0x1000e0a982f0001 2024-12-10T04:45:05,048 DEBUG [RS:0;6578523f4421:42295 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T04:45:05,048 DEBUG [RS:0;6578523f4421:42295 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6578523f4421,42295,1733805904421 2024-12-10T04:45:05,048 DEBUG [RS:0;6578523f4421:42295 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6578523f4421,42295,1733805904421' 2024-12-10T04:45:05,048 DEBUG [RS:0;6578523f4421:42295 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T04:45:05,049 DEBUG [RS:0;6578523f4421:42295 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T04:45:05,050 DEBUG [RS:0;6578523f4421:42295 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T04:45:05,050 INFO [RS:1;6578523f4421:46207 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,050 DEBUG [RS:0;6578523f4421:42295 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T04:45:05,050 DEBUG [RS:0;6578523f4421:42295 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6578523f4421,42295,1733805904421 2024-12-10T04:45:05,050 INFO [RS:1;6578523f4421:46207 {}] regionserver.HRegionServer(1482): Serving as 6578523f4421,46207,1733805904462, RpcServer on 6578523f4421/172.17.0.2:46207, sessionid=0x1000e0a982f0002 2024-12-10T04:45:05,050 DEBUG [RS:0;6578523f4421:42295 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6578523f4421,42295,1733805904421' 2024-12-10T04:45:05,050 DEBUG [RS:0;6578523f4421:42295 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T04:45:05,050 DEBUG [RS:1;6578523f4421:46207 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T04:45:05,050 DEBUG [RS:1;6578523f4421:46207 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6578523f4421,46207,1733805904462 2024-12-10T04:45:05,050 DEBUG [RS:1;6578523f4421:46207 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6578523f4421,46207,1733805904462' 2024-12-10T04:45:05,050 DEBUG [RS:1;6578523f4421:46207 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T04:45:05,051 DEBUG [RS:0;6578523f4421:42295 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T04:45:05,051 DEBUG [RS:1;6578523f4421:46207 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T04:45:05,051 DEBUG [RS:0;6578523f4421:42295 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T04:45:05,051 INFO [RS:0;6578523f4421:42295 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T04:45:05,051 INFO [RS:0;6578523f4421:42295 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T04:45:05,052 DEBUG [RS:1;6578523f4421:46207 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T04:45:05,052 DEBUG [RS:1;6578523f4421:46207 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T04:45:05,052 DEBUG [RS:1;6578523f4421:46207 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6578523f4421,46207,1733805904462 2024-12-10T04:45:05,052 DEBUG [RS:1;6578523f4421:46207 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6578523f4421,46207,1733805904462' 2024-12-10T04:45:05,052 DEBUG [RS:1;6578523f4421:46207 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T04:45:05,052 DEBUG [RS:1;6578523f4421:46207 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T04:45:05,052 DEBUG [RS:1;6578523f4421:46207 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T04:45:05,052 INFO [RS:1;6578523f4421:46207 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T04:45:05,053 INFO [RS:1;6578523f4421:46207 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T04:45:05,096 WARN [6578523f4421:42569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-10T04:45:05,152 INFO [RS:0;6578523f4421:42295 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T04:45:05,153 INFO [RS:0;6578523f4421:42295 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6578523f4421%2C42295%2C1733805904421, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,42295,1733805904421, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/oldWALs, maxLogs=55 2024-12-10T04:45:05,153 INFO [RS:1;6578523f4421:46207 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T04:45:05,154 INFO [RS:1;6578523f4421:46207 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6578523f4421%2C46207%2C1733805904462, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,46207,1733805904462, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/oldWALs, maxLogs=55 2024-12-10T04:45:05,171 INFO [RS:0;6578523f4421:42295 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,42295,1733805904421/6578523f4421%2C42295%2C1733805904421.1733805905155 2024-12-10T04:45:05,171 INFO [RS:1;6578523f4421:46207 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,46207,1733805904462/6578523f4421%2C46207%2C1733805904462.1733805905156 2024-12-10T04:45:05,171 DEBUG [RS:0;6578523f4421:42295 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T04:45:05,172 DEBUG [RS:1;6578523f4421:46207 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T04:45:05,346 DEBUG [6578523f4421:42569 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-10T04:45:05,347 DEBUG [6578523f4421:42569 {}] balancer.BalancerClusterState(204): Hosts are {6578523f4421=0} racks are {/default-rack=0} 2024-12-10T04:45:05,347 DEBUG [6578523f4421:42569 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T04:45:05,347 DEBUG [6578523f4421:42569 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T04:45:05,347 DEBUG [6578523f4421:42569 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T04:45:05,347 DEBUG [6578523f4421:42569 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T04:45:05,348 INFO [6578523f4421:42569 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T04:45:05,348 INFO [6578523f4421:42569 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T04:45:05,348 DEBUG [6578523f4421:42569 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T04:45:05,348 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6578523f4421,46207,1733805904462 2024-12-10T04:45:05,349 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6578523f4421,46207,1733805904462, state=OPENING 2024-12-10T04:45:05,360 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-10T04:45:05,371 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:05,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:05,371 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T04:45:05,372 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T04:45:05,372 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6578523f4421,46207,1733805904462}] 2024-12-10T04:45:05,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46207-0x1000e0a982f0002, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:05,377 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T04:45:05,377 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T04:45:05,525 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T04:45:05,528 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45493, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T04:45:05,530 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-10T04:45:05,531 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T04:45:05,531 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T04:45:05,533 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6578523f4421%2C46207%2C1733805904462.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,46207,1733805904462, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/oldWALs, maxLogs=55 2024-12-10T04:45:05,558 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,46207,1733805904462/6578523f4421%2C46207%2C1733805904462.meta.1733805905533.meta 2024-12-10T04:45:05,561 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T04:45:05,561 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T04:45:05,562 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T04:45:05,562 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T04:45:05,562 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T04:45:05,562 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T04:45:05,562 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T04:45:05,562 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-10T04:45:05,562 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-10T04:45:05,563 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-10T04:45:05,564 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T04:45:05,564 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:05,565 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:05,565 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-10T04:45:05,566 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T04:45:05,566 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:05,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:05,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-10T04:45:05,568 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T04:45:05,568 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:05,568 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:05,569 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-10T04:45:05,570 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T04:45:05,570 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:05,570 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:05,570 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T04:45:05,571 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740 2024-12-10T04:45:05,572 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740 2024-12-10T04:45:05,572 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T04:45:05,572 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T04:45:05,572 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T04:45:05,574 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T04:45:05,574 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64661057, jitterRate=-0.03647516667842865}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T04:45:05,574 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-10T04:45:05,575 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733805905563Writing region info on filesystem at 1733805905563Initializing all the Stores at 1733805905563Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733805905563Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733805905563Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733805905563Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733805905563Cleaning up temporary data from old regions at 1733805905572 (+9 ms)Running coprocessor post-open hooks at 1733805905574 (+2 ms)Region opened successfully at 1733805905575 (+1 ms) 2024-12-10T04:45:05,576 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733805905524 2024-12-10T04:45:05,579 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T04:45:05,579 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-10T04:45:05,580 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=6578523f4421,46207,1733805904462 2024-12-10T04:45:05,582 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6578523f4421,46207,1733805904462, state=OPEN 2024-12-10T04:45:05,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46207-0x1000e0a982f0002, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T04:45:05,592 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T04:45:05,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T04:45:05,592 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T04:45:05,592 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6578523f4421,46207,1733805904462 2024-12-10T04:45:05,592 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T04:45:05,592 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T04:45:05,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-10T04:45:05,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6578523f4421,46207,1733805904462 in 220 msec 2024-12-10T04:45:05,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-10T04:45:05,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 651 msec 2024-12-10T04:45:05,599 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T04:45:05,599 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-10T04:45:05,605 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T04:45:05,606 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6578523f4421,46207,1733805904462, seqNum=-1] 2024-12-10T04:45:05,606 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T04:45:05,608 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36999, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T04:45:05,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 736 msec 2024-12-10T04:45:05,617 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733805905617, completionTime=-1 2024-12-10T04:45:05,618 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-10T04:45:05,618 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-10T04:45:05,621 INFO [master/6578523f4421:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-10T04:45:05,621 INFO [master/6578523f4421:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733805965621 2024-12-10T04:45:05,621 INFO [master/6578523f4421:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733806025621 2024-12-10T04:45:05,622 INFO [master/6578523f4421:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-12-10T04:45:05,622 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-10T04:45:05,625 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,42569,1733805904248-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,625 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,42569,1733805904248-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,625 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,42569,1733805904248-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,625 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-6578523f4421:42569, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,625 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,625 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,628 DEBUG [master/6578523f4421:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-10T04:45:05,633 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.094sec 2024-12-10T04:45:05,637 INFO [master/6578523f4421:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-10T04:45:05,637 INFO [master/6578523f4421:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-10T04:45:05,637 INFO [master/6578523f4421:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-10T04:45:05,637 INFO [master/6578523f4421:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-10T04:45:05,637 INFO [master/6578523f4421:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-10T04:45:05,637 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,42569,1733805904248-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T04:45:05,637 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,42569,1733805904248-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-10T04:45:05,641 DEBUG [master/6578523f4421:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-10T04:45:05,641 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-10T04:45:05,641 INFO [master/6578523f4421:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,42569,1733805904248-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:05,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143f7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T04:45:05,704 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6578523f4421,42569,-1 for getting cluster id 2024-12-10T04:45:05,704 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T04:45:05,707 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98590c83-ab1a-4d90-943c-331e4788e621' 2024-12-10T04:45:05,708 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T04:45:05,708 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98590c83-ab1a-4d90-943c-331e4788e621" 2024-12-10T04:45:05,709 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70d1478f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T04:45:05,709 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6578523f4421,42569,-1] 2024-12-10T04:45:05,709 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T04:45:05,710 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T04:45:05,713 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T04:45:05,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af0bbc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T04:45:05,715 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T04:45:05,717 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6578523f4421,46207,1733805904462, seqNum=-1] 2024-12-10T04:45:05,718 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T04:45:05,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T04:45:05,723 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=6578523f4421,42569,1733805904248 2024-12-10T04:45:05,723 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T04:45:05,725 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 6578523f4421,42569,1733805904248 2024-12-10T04:45:05,725 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@799e19f8 2024-12-10T04:45:05,726 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T04:45:05,730 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T04:45:05,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42569 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733805905723', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T04:45:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42569 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733805905723 2024-12-10T04:45:05,734 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733805905723 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T04:45:05,734 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:05,735 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42569 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733805905723" procId is: 4 2024-12-10T04:45:05,736 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733805905723 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T04:45:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T04:45:05,754 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733805905723-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 146881db9cca4e9e603d9b917ae4c321, NAME => 'testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733805905723', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570 2024-12-10T04:45:05,760 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733805905723-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T04:45:05,761 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733805905723-pool-0 {}] regionserver.HRegion(1722): Closing 146881db9cca4e9e603d9b917ae4c321, disabling compactions & flushes 2024-12-10T04:45:05,761 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733805905723-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. 2024-12-10T04:45:05,761 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733805905723-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. 2024-12-10T04:45:05,761 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733805905723-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. after waiting 0 ms 2024-12-10T04:45:05,761 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733805905723-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. 2024-12-10T04:45:05,761 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733805905723-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. 2024-12-10T04:45:05,761 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733805905723-pool-0 {}] regionserver.HRegion(1676): Region close journal for 146881db9cca4e9e603d9b917ae4c321: Waiting for close lock at 1733805905761Disabling compacts and flushes for region at 1733805905761Disabling writes for close at 1733805905761Writing region close event to WAL at 1733805905761Closed at 1733805905761 2024-12-10T04:45:05,762 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733805905723 execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T04:45:05,763 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733805905762"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733805905762"}]},"ts":"1733805905762"} 2024-12-10T04:45:05,766 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T04:45:05,767 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733805905723 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T04:45:05,768 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733805905723","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733805905768"}]},"ts":"1733805905768"} 2024-12-10T04:45:05,771 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733805905723, state=ENABLING in hbase:meta 2024-12-10T04:45:05,771 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {6578523f4421=0} racks are {/default-rack=0} 2024-12-10T04:45:05,772 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T04:45:05,772 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T04:45:05,772 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T04:45:05,772 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T04:45:05,772 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T04:45:05,772 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T04:45:05,772 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T04:45:05,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733805905723, region=146881db9cca4e9e603d9b917ae4c321, ASSIGN}] 2024-12-10T04:45:05,774 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733805905723, region=146881db9cca4e9e603d9b917ae4c321, ASSIGN 2024-12-10T04:45:05,776 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733805905723, region=146881db9cca4e9e603d9b917ae4c321, ASSIGN; state=OFFLINE, location=6578523f4421,46207,1733805904462; forceNewPlan=false, retain=false 2024-12-10T04:45:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T04:45:05,926 INFO [6578523f4421:42569 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T04:45:05,927 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=146881db9cca4e9e603d9b917ae4c321, regionState=OPENING, regionLocation=6578523f4421,46207,1733805904462 2024-12-10T04:45:05,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733805905723, region=146881db9cca4e9e603d9b917ae4c321, ASSIGN because future has completed 2024-12-10T04:45:05,931 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 146881db9cca4e9e603d9b917ae4c321, server=6578523f4421,46207,1733805904462}] 2024-12-10T04:45:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T04:45:06,088 INFO [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. 2024-12-10T04:45:06,088 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 146881db9cca4e9e603d9b917ae4c321, NAME => 'testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321.', STARTKEY => '', ENDKEY => ''} 2024-12-10T04:45:06,089 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733805905723 146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:06,089 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T04:45:06,090 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:06,090 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:06,091 INFO [StoreOpener-146881db9cca4e9e603d9b917ae4c321-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:06,096 INFO [StoreOpener-146881db9cca4e9e603d9b917ae4c321-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 146881db9cca4e9e603d9b917ae4c321 columnFamilyName family 2024-12-10T04:45:06,096 DEBUG [StoreOpener-146881db9cca4e9e603d9b917ae4c321-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:06,097 INFO [StoreOpener-146881db9cca4e9e603d9b917ae4c321-1 {}] regionserver.HStore(327): Store=146881db9cca4e9e603d9b917ae4c321/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T04:45:06,097 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:06,098 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:06,098 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:06,098 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:06,099 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:06,099 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:06,105 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T04:45:06,106 INFO [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 146881db9cca4e9e603d9b917ae4c321; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69907981, jitterRate=0.04171009361743927}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T04:45:06,106 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:06,106 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 146881db9cca4e9e603d9b917ae4c321: Running coprocessor pre-open hook at 1733805906090Writing region info on filesystem at 1733805906090Initializing all the Stores at 1733805906090Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733805906090Cleaning up temporary data from old regions at 1733805906099 (+9 ms)Running coprocessor post-open hooks at 1733805906106 (+7 ms)Region opened successfully at 1733805906106 2024-12-10T04:45:06,107 INFO [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321., pid=6, masterSystemTime=1733805906085 2024-12-10T04:45:06,111 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. 2024-12-10T04:45:06,111 INFO [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. 2024-12-10T04:45:06,111 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=146881db9cca4e9e603d9b917ae4c321, regionState=OPEN, openSeqNum=2, regionLocation=6578523f4421,46207,1733805904462 2024-12-10T04:45:06,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 146881db9cca4e9e603d9b917ae4c321, server=6578523f4421,46207,1733805904462 because future has completed 2024-12-10T04:45:06,120 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-10T04:45:06,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 146881db9cca4e9e603d9b917ae4c321, server=6578523f4421,46207,1733805904462 in 186 msec 2024-12-10T04:45:06,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-10T04:45:06,124 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733805905723, region=146881db9cca4e9e603d9b917ae4c321, ASSIGN in 349 msec 2024-12-10T04:45:06,125 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733805905723 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T04:45:06,125 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733805905723","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733805906125"}]},"ts":"1733805906125"} 2024-12-10T04:45:06,128 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733805905723, state=ENABLED in hbase:meta 2024-12-10T04:45:06,130 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733805905723 execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T04:45:06,132 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733805905723 in 398 msec 2024-12-10T04:45:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T04:45:06,371 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733805905723 completed 2024-12-10T04:45:06,371 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733805905723 get assigned. Timeout = 60000ms 2024-12-10T04:45:06,371 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T04:45:06,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733805905723 assigned to meta. Checking AM states. 2024-12-10T04:45:06,376 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T04:45:06,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733805905723 assigned. 2024-12-10T04:45:06,381 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733805905723', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321., hostname=6578523f4421,46207,1733805904462, seqNum=2] 2024-12-10T04:45:06,388 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 146881db9cca4e9e603d9b917ae4c321 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-10T04:45:06,399 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8c32f883261b45f8a311a7e199b8dc1b 2024-12-10T04:45:06,417 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321/.tmp/family/8c32f883261b45f8a311a7e199b8dc1b is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-10T04:45:06,417 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321/.tmp/family/8c32f883261b45f8a311a7e199b8dc1b 2024-12-10T04:45:06,422 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321/.tmp/family/8c32f883261b45f8a311a7e199b8dc1b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321/family/8c32f883261b45f8a311a7e199b8dc1b 2024-12-10T04:45:06,424 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321/family/8c32f883261b45f8a311a7e199b8dc1b, delay=904 ms 2024-12-10T04:45:06,426 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321/family/8c32f883261b45f8a311a7e199b8dc1b, entries=2, sequenceid=6, filesize=4.9 K 2024-12-10T04:45:06,427 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 146881db9cca4e9e603d9b917ae4c321 in 38ms, sequenceid=6, compaction requested=false 2024-12-10T04:45:06,427 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733805905723' 2024-12-10T04:45:06,428 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 146881db9cca4e9e603d9b917ae4c321: 2024-12-10T04:45:06,445 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T04:45:07,325 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-10T04:45:07,326 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-10T04:45:07,328 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1733805905723 2024-12-10T04:45:07,328 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1733805905723 Metrics about Tables on a single HBase RegionServer 2024-12-10T04:45:07,329 DEBUG [hfile-prefetch-1733805903608 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8c32f883261b45f8a311a7e199b8dc1b_0. 2024-12-10T04:45:07,329 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T04:45:07,329 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-10T04:45:07,330 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-10T04:45:07,330 DEBUG [hfile-prefetch-1733805903608 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8c32f883261b45f8a311a7e199b8dc1b_117. 2024-12-10T04:45:07,330 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-10T04:45:07,331 DEBUG [hfile-prefetch-1733805903608 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321/family/8c32f883261b45f8a311a7e199b8dc1b, with total blocks 2, and data blocks 1 2024-12-10T04:45:07,331 DEBUG [hfile-prefetch-1733805903608 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-10T04:45:07,331 DEBUG [hfile-prefetch-1733805903608 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321/family/8c32f883261b45f8a311a7e199b8dc1b, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-10T04:45:07,331 DEBUG [hfile-prefetch-1733805903608 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321/family/8c32f883261b45f8a311a7e199b8dc1b. Will sleep 100ms and try the verification again. 2024-12-10T04:45:07,391 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T04:45:07,391 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T04:45:07,391 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T04:45:07,391 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T04:45:07,405 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-10T04:45:07,405 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733805907391 2024-12-10T04:45:07,405 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T04:45:07,428 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;6578523f4421:46207,5,FailOnTimeoutGroup] 2024-12-10T04:45:07,428 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6578523f4421,46207,1733805904462' ***** 2024-12-10T04:45:07,428 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-12-10T04:45:07,429 INFO [RS:1;6578523f4421:46207 {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T04:45:07,429 INFO [RS:1;6578523f4421:46207 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630/bucket.persistence 2024-12-10T04:45:07,429 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T04:45:07,429 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T04:45:07,429 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T04:45:07,429 INFO [RS:1;6578523f4421:46207 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T04:45:07,429 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T04:45:07,431 DEBUG [hfile-prefetch-1733805903608 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321/family/8c32f883261b45f8a311a7e199b8dc1b, with total blocks 2, and data blocks 1 2024-12-10T04:45:07,431 DEBUG [hfile-prefetch-1733805903608 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-12-10T04:45:07,431 DEBUG [hfile-prefetch-1733805903608 {}] bucket.BucketCache(2293): found block for file 8c32f883261b45f8a311a7e199b8dc1b in the backing map. Acquiring read lock for offset 0 2024-12-10T04:45:07,432 DEBUG [hfile-prefetch-1733805903608 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321/family/8c32f883261b45f8a311a7e199b8dc1b has now been fully cached. 2024-12-10T04:45:07,432 DEBUG [hfile-prefetch-1733805903608 {}] hfile.PrefetchExecutor(121): Prefetch completed for 8c32f883261b45f8a311a7e199b8dc1b 2024-12-10T04:45:07,439 DEBUG [RS:1;6578523f4421:46207 {}] bucket.BucketCache(1487): Thread RS:1;6578523f4421:46207 started persisting bucket cache to file 2024-12-10T04:45:07,440 DEBUG [RS:1;6578523f4421:46207 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T04:45:07,440 DEBUG [RS:1;6578523f4421:46207 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-12-10T04:45:07,446 DEBUG [RS:1;6578523f4421:46207 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-12-10T04:45:07,446 DEBUG [RS:1;6578523f4421:46207 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1733805907439 2024-12-10T04:45:07,446 DEBUG [RS:1;6578523f4421:46207 {}] bucket.BucketCache(1509): Thread RS:1;6578523f4421:46207 finished persisting bucket cache to file, renaming 2024-12-10T04:45:07,447 INFO [RS:1;6578523f4421:46207 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T04:45:07,447 INFO [RS:1;6578523f4421:46207 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T04:45:07,447 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T04:45:07,447 INFO [RS:1;6578523f4421:46207 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T04:45:07,447 INFO [RS:1;6578523f4421:46207 {}] regionserver.HRegionServer(3091): Received CLOSE for 146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:07,447 INFO [RS:1;6578523f4421:46207 {}] regionserver.HRegionServer(959): stopping server 6578523f4421,46207,1733805904462 2024-12-10T04:45:07,447 INFO [RS:1;6578523f4421:46207 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T04:45:07,448 INFO [RS:1;6578523f4421:46207 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;6578523f4421:46207. 2024-12-10T04:45:07,448 INFO [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T04:45:07,448 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 146881db9cca4e9e603d9b917ae4c321, disabling compactions & flushes 2024-12-10T04:45:07,448 DEBUG [RS:1;6578523f4421:46207 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T04:45:07,448 DEBUG [RS:1;6578523f4421:46207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T04:45:07,448 INFO [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. 2024-12-10T04:45:07,448 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. 2024-12-10T04:45:07,448 INFO [RS:1;6578523f4421:46207 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T04:45:07,448 INFO [RS:1;6578523f4421:46207 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T04:45:07,448 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. after waiting 0 ms 2024-12-10T04:45:07,448 INFO [RS:1;6578523f4421:46207 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T04:45:07,448 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. 2024-12-10T04:45:07,448 INFO [RS:1;6578523f4421:46207 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-10T04:45:07,449 INFO [RS:1;6578523f4421:46207 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-10T04:45:07,449 DEBUG [RS:1;6578523f4421:46207 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 146881db9cca4e9e603d9b917ae4c321=testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321.} 2024-12-10T04:45:07,449 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T04:45:07,449 DEBUG [RS:1;6578523f4421:46207 {}] regionserver.HRegionServer(1351): Waiting on 146881db9cca4e9e603d9b917ae4c321, 1588230740 2024-12-10T04:45:07,449 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T04:45:07,450 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T04:45:07,450 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T04:45:07,450 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T04:45:07,450 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T04:45:07,450 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-12-10T04:45:07,455 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-10T04:45:07,456 INFO [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. 2024-12-10T04:45:07,456 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 146881db9cca4e9e603d9b917ae4c321: Waiting for close lock at 1733805907448Running coprocessor pre-close hooks at 1733805907448Disabling compacts and flushes for region at 1733805907448Disabling writes for close at 1733805907448Writing region close event to WAL at 1733805907450 (+2 ms)Running coprocessor post-close hooks at 1733805907456 (+6 ms)Closed at 1733805907456 2024-12-10T04:45:07,456 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. 2024-12-10T04:45:07,460 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6acfe5b9dd914a45aa8b4caa2fa3b84d 2024-12-10T04:45:07,473 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/info/6acfe5b9dd914a45aa8b4caa2fa3b84d is 209, key is testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321./info:regioninfo/1733805906111/Put/seqid=0 2024-12-10T04:45:07,473 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/info/6acfe5b9dd914a45aa8b4caa2fa3b84d 2024-12-10T04:45:07,480 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3e5b7752e9bd448397a2240c3eb8a7e9 2024-12-10T04:45:07,491 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/ns/3e5b7752e9bd448397a2240c3eb8a7e9 is 43, key is default/ns:d/1733805905609/Put/seqid=0 2024-12-10T04:45:07,492 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/ns/3e5b7752e9bd448397a2240c3eb8a7e9 2024-12-10T04:45:07,498 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 761c64c62e4740069a8905809e21b091 2024-12-10T04:45:07,509 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/table/761c64c62e4740069a8905809e21b091 is 80, key is testBlockEvictionOnGracefulStop1733805905723/table:state/1733805906125/Put/seqid=0 2024-12-10T04:45:07,510 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/table/761c64c62e4740069a8905809e21b091 2024-12-10T04:45:07,512 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/info/6acfe5b9dd914a45aa8b4caa2fa3b84d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/info/6acfe5b9dd914a45aa8b4caa2fa3b84d 2024-12-10T04:45:07,514 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/info/6acfe5b9dd914a45aa8b4caa2fa3b84d, delay=974 ms 2024-12-10T04:45:07,515 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/info/6acfe5b9dd914a45aa8b4caa2fa3b84d, entries=10, sequenceid=11, filesize=7.0 K 2024-12-10T04:45:07,515 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/ns/3e5b7752e9bd448397a2240c3eb8a7e9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/ns/3e5b7752e9bd448397a2240c3eb8a7e9 2024-12-10T04:45:07,516 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/ns/3e5b7752e9bd448397a2240c3eb8a7e9, delay=956 ms 2024-12-10T04:45:07,517 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/ns/3e5b7752e9bd448397a2240c3eb8a7e9, entries=2, sequenceid=11, filesize=5.0 K 2024-12-10T04:45:07,518 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/table/761c64c62e4740069a8905809e21b091 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/table/761c64c62e4740069a8905809e21b091 2024-12-10T04:45:07,518 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/table/761c64c62e4740069a8905809e21b091, delay=969 ms 2024-12-10T04:45:07,519 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/table/761c64c62e4740069a8905809e21b091, entries=2, sequenceid=11, filesize=5.3 K 2024-12-10T04:45:07,520 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 70ms, sequenceid=11, compaction requested=false 2024-12-10T04:45:07,521 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/info/6acfe5b9dd914a45aa8b4caa2fa3b84d 2024-12-10T04:45:07,521 INFO [regionserver/6578523f4421:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T04:45:07,521 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/ns/3e5b7752e9bd448397a2240c3eb8a7e9 2024-12-10T04:45:07,522 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/table/761c64c62e4740069a8905809e21b091 2024-12-10T04:45:07,525 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-10T04:45:07,526 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T04:45:07,526 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T04:45:07,526 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733805907449Running coprocessor pre-close hooks at 1733805907449Disabling compacts and flushes for region at 1733805907449Disabling writes for close at 1733805907450 (+1 ms)Obtaining lock to block concurrent updates at 1733805907450Preparing flush snapshotting stores in 1588230740 at 1733805907450Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1733805907451 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733805907451Flushing 1588230740/info: creating writer at 1733805907451Flushing 1588230740/info: appending metadata at 1733805907472 (+21 ms)Flushing 1588230740/info: closing flushed file at 1733805907472Flushing 1588230740/ns: creating writer at 1733805907475 (+3 ms)Flushing 1588230740/ns: appending metadata at 1733805907491 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733805907491Flushing 1588230740/table: creating writer at 1733805907494 (+3 ms)Flushing 1588230740/table: appending metadata at 1733805907509 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733805907509Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@155e6a85: reopening flushed file at 1733805907512 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32a89727: reopening flushed file at 1733805907515 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e578316: reopening flushed file at 1733805907517 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 70ms, sequenceid=11, compaction requested=false at 1733805907520 (+3 ms)Writing region close event to WAL at 1733805907522 (+2 ms)Running coprocessor post-close hooks at 1733805907525 (+3 ms)Closed at 1733805907526 (+1 ms) 2024-12-10T04:45:07,526 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T04:45:07,650 INFO [RS:1;6578523f4421:46207 {}] regionserver.HRegionServer(976): stopping server 6578523f4421,46207,1733805904462; all regions closed. 2024-12-10T04:45:07,651 DEBUG [RS:1;6578523f4421:46207 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/oldWALs 2024-12-10T04:45:07,651 INFO [RS:1;6578523f4421:46207 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6578523f4421%2C46207%2C1733805904462.meta:.meta(num 1733805905533) 2024-12-10T04:45:07,653 DEBUG [RS:1;6578523f4421:46207 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/oldWALs 2024-12-10T04:45:07,653 INFO [RS:1;6578523f4421:46207 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6578523f4421%2C46207%2C1733805904462:(num 1733805905156) 2024-12-10T04:45:07,653 DEBUG [RS:1;6578523f4421:46207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T04:45:07,653 INFO [RS:1;6578523f4421:46207 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T04:45:07,653 INFO [RS:1;6578523f4421:46207 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T04:45:07,653 INFO [RS:1;6578523f4421:46207 {}] hbase.ChoreService(370): Chore service for: regionserver/6578523f4421:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T04:45:07,653 INFO [RS:1;6578523f4421:46207 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T04:45:07,653 INFO [regionserver/6578523f4421:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T04:45:07,654 INFO [RS:1;6578523f4421:46207 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46207 2024-12-10T04:45:07,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46207-0x1000e0a982f0002, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6578523f4421,46207,1733805904462 2024-12-10T04:45:07,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T04:45:07,666 INFO [RS:1;6578523f4421:46207 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T04:45:07,667 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6578523f4421,46207,1733805904462] 2024-12-10T04:45:07,686 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6578523f4421,46207,1733805904462 already deleted, retry=false 2024-12-10T04:45:07,687 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 6578523f4421,46207,1733805904462 on 6578523f4421,42569,1733805904248 2024-12-10T04:45:07,693 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 6578523f4421,46207,1733805904462, splitWal=true, meta=true 2024-12-10T04:45:07,697 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 6578523f4421,46207,1733805904462 (carryingMeta=true) 6578523f4421,46207,1733805904462/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@715fef0a[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-10T04:45:07,697 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 6578523f4421,46207,1733805904462, splitWal=true, meta=true 2024-12-10T04:45:07,698 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 6578523f4421,46207,1733805904462, splitWal=true, meta=true, isMeta: true 2024-12-10T04:45:07,699 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,46207,1733805904462-splitting 2024-12-10T04:45:07,699 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,46207,1733805904462-splitting dir is empty, no logs to split. 2024-12-10T04:45:07,699 INFO [PEWorker-1 {}] master.SplitWALManager(105): 6578523f4421,46207,1733805904462 WAL count=0, meta=true 2024-12-10T04:45:07,702 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,46207,1733805904462-splitting dir is empty, no logs to split. 2024-12-10T04:45:07,702 INFO [PEWorker-1 {}] master.SplitWALManager(105): 6578523f4421,46207,1733805904462 WAL count=0, meta=true 2024-12-10T04:45:07,702 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 6578523f4421,46207,1733805904462 WAL splitting is done? wals=0, meta=true 2024-12-10T04:45:07,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T04:45:07,706 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T04:45:07,708 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-10T04:45:07,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46207-0x1000e0a982f0002, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T04:45:07,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46207-0x1000e0a982f0002, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T04:45:07,776 INFO [RS:1;6578523f4421:46207 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T04:45:07,776 INFO [RS:1;6578523f4421:46207 {}] regionserver.HRegionServer(1031): Exiting; stopping=6578523f4421,46207,1733805904462; zookeeper connection closed. 2024-12-10T04:45:07,776 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@c138228 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@c138228 2024-12-10T04:45:07,858 DEBUG [6578523f4421:42569 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-10T04:45:07,859 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6578523f4421,42295,1733805904421 2024-12-10T04:45:07,860 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6578523f4421,42295,1733805904421, state=OPENING 2024-12-10T04:45:07,865 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T04:45:07,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T04:45:07,865 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T04:45:07,865 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T04:45:07,865 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T04:45:07,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6578523f4421,42295,1733805904421}] 2024-12-10T04:45:07,948 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6578523f4421:0 server-side Connection retries=45 2024-12-10T04:45:07,948 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T04:45:07,948 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T04:45:07,948 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T04:45:07,948 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T04:45:07,948 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T04:45:07,948 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T04:45:07,948 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T04:45:07,949 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43039 2024-12-10T04:45:07,951 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43039 connecting to ZooKeeper ensemble=127.0.0.1:57720 2024-12-10T04:45:07,951 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:45:07,951 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:45:07,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:430390x0, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T04:45:07,961 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:430390x0, quorum=127.0.0.1:57720, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-10T04:45:07,961 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-10T04:45:07,961 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43039-0x1000e0a982f0003 connected 2024-12-10T04:45:07,961 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T04:45:07,965 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630/bucket.cache 2024-12-10T04:45:07,965 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T04:45:07,965 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T04:45:07,965 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-10T04:45:07,969 INFO [Thread-207 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T04:45:07,970 INFO [Thread-207 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-10T04:45:07,972 INFO [Thread-207 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-12-10T04:45:07,978 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T04:45:07,989 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T04:45:07,989 DEBUG [Thread-207 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-10T04:45:07,989 DEBUG [Thread-207 {}] bucket.BucketCache(1584): Cached File Entry:<8c32f883261b45f8a311a7e199b8dc1b,<146881db9cca4e9e603d9b917ae4c321,158>> 2024-12-10T04:45:07,990 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:43039-0x1000e0a982f0003, quorum=127.0.0.1:57720, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T04:45:07,990 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T04:45:07,990 INFO [Thread-207 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-12-10T04:45:07,990 INFO [Thread-207 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630/bucket.persistence is complete. 2024-12-10T04:45:07,991 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43039-0x1000e0a982f0003, quorum=127.0.0.1:57720, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T04:45:08,030 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T04:45:08,049 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43039 2024-12-10T04:45:08,054 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48983, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T04:45:08,065 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43039 2024-12-10T04:45:08,069 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-10T04:45:08,069 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T04:45:08,069 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T04:45:08,070 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6578523f4421%2C42295%2C1733805904421.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,42295,1733805904421, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/oldWALs, maxLogs=55 2024-12-10T04:45:08,074 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43039 2024-12-10T04:45:08,076 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43039 2024-12-10T04:45:08,076 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43039 2024-12-10T04:45:08,082 INFO [RS:2;6578523f4421:43039 {}] regionserver.HRegionServer(746): ClusterId : 98590c83-ab1a-4d90-943c-331e4788e621 2024-12-10T04:45:08,082 DEBUG [RS:2;6578523f4421:43039 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T04:45:08,099 DEBUG [RS:2;6578523f4421:43039 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T04:45:08,099 DEBUG [RS:2;6578523f4421:43039 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T04:45:08,100 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,42295,1733805904421/6578523f4421%2C42295%2C1733805904421.meta.1733805908071.meta 2024-12-10T04:45:08,100 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T04:45:08,101 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T04:45:08,101 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T04:45:08,101 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T04:45:08,101 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T04:45:08,102 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T04:45:08,102 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T04:45:08,102 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-10T04:45:08,102 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-10T04:45:08,108 DEBUG [RS:2;6578523f4421:43039 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T04:45:08,109 DEBUG [RS:2;6578523f4421:43039 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5090f6ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6578523f4421/172.17.0.2:0 2024-12-10T04:45:08,113 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-10T04:45:08,114 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T04:45:08,114 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:08,122 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/info/6acfe5b9dd914a45aa8b4caa2fa3b84d, delay=951 ms 2024-12-10T04:45:08,123 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/info/6acfe5b9dd914a45aa8b4caa2fa3b84d 2024-12-10T04:45:08,123 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:08,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-10T04:45:08,124 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T04:45:08,125 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:08,127 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/ns/3e5b7752e9bd448397a2240c3eb8a7e9, delay=978 ms 2024-12-10T04:45:08,129 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/ns/3e5b7752e9bd448397a2240c3eb8a7e9 2024-12-10T04:45:08,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:08,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-10T04:45:08,130 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T04:45:08,130 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:08,131 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:08,131 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-10T04:45:08,132 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T04:45:08,132 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:08,133 DEBUG [RS:2;6578523f4421:43039 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;6578523f4421:43039 2024-12-10T04:45:08,133 INFO [RS:2;6578523f4421:43039 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T04:45:08,133 INFO [RS:2;6578523f4421:43039 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T04:45:08,133 DEBUG [RS:2;6578523f4421:43039 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T04:45:08,134 INFO [RS:2;6578523f4421:43039 {}] regionserver.HRegionServer(2659): reportForDuty to master=6578523f4421,42569,1733805904248 with port=43039, startcode=1733805907947 2024-12-10T04:45:08,134 DEBUG [RS:2;6578523f4421:43039 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T04:45:08,138 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/table/761c64c62e4740069a8905809e21b091, delay=909 ms 2024-12-10T04:45:08,140 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/table/761c64c62e4740069a8905809e21b091 2024-12-10T04:45:08,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T04:45:08,141 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T04:45:08,142 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740 2024-12-10T04:45:08,143 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740 2024-12-10T04:45:08,143 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T04:45:08,143 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T04:45:08,144 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T04:45:08,146 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T04:45:08,146 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71036280, jitterRate=0.05852305889129639}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T04:45:08,147 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-10T04:45:08,148 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733805908102Writing region info on filesystem at 1733805908102Initializing all the Stores at 1733805908103 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733805908103Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733805908113 (+10 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733805908113Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733805908113Cleaning up temporary data from old regions at 1733805908144 (+31 ms)Running coprocessor post-open hooks at 1733805908147 (+3 ms)Region opened successfully at 1733805908148 (+1 ms) 2024-12-10T04:45:08,149 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1733805908029 2024-12-10T04:45:08,151 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47781, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T04:45:08,152 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42569 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6578523f4421,43039,1733805907947 2024-12-10T04:45:08,152 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42569 {}] master.ServerManager(517): Registering regionserver=6578523f4421,43039,1733805907947 2024-12-10T04:45:08,153 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=6578523f4421,42295,1733805904421 2024-12-10T04:45:08,153 DEBUG [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T04:45:08,154 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6578523f4421,42295,1733805904421, state=OPEN 2024-12-10T04:45:08,154 INFO [RS_OPEN_META-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-10T04:45:08,155 DEBUG [RS:2;6578523f4421:43039 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570 2024-12-10T04:45:08,155 DEBUG [RS:2;6578523f4421:43039 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-10T04:45:08,155 DEBUG [RS:2;6578523f4421:43039 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T04:45:08,160 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T04:45:08,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43039-0x1000e0a982f0003, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T04:45:08,160 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T04:45:08,160 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T04:45:08,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T04:45:08,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T04:45:08,161 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6578523f4421,42295,1733805904421 2024-12-10T04:45:08,164 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-10T04:45:08,164 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6578523f4421,42295,1733805904421 in 296 msec 2024-12-10T04:45:08,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-10T04:45:08,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 460 msec 2024-12-10T04:45:08,167 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 6578523f4421,46207,1733805904462 had 2 regions 2024-12-10T04:45:08,169 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 6578523f4421,46207,1733805904462, splitWal=true, meta=true, isMeta: false 2024-12-10T04:45:08,170 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,46207,1733805904462-splitting dir is empty, no logs to split. 2024-12-10T04:45:08,170 INFO [PEWorker-3 {}] master.SplitWALManager(105): 6578523f4421,46207,1733805904462 WAL count=0, meta=false 2024-12-10T04:45:08,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T04:45:08,172 DEBUG [RS:2;6578523f4421:43039 {}] zookeeper.ZKUtil(111): regionserver:43039-0x1000e0a982f0003, quorum=127.0.0.1:57720, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6578523f4421,43039,1733805907947 2024-12-10T04:45:08,172 WARN [RS:2;6578523f4421:43039 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T04:45:08,172 INFO [RS:2;6578523f4421:43039 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T04:45:08,172 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,46207,1733805904462-splitting dir is empty, no logs to split. 2024-12-10T04:45:08,172 INFO [PEWorker-3 {}] master.SplitWALManager(105): 6578523f4421,46207,1733805904462 WAL count=0, meta=false 2024-12-10T04:45:08,172 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 6578523f4421,46207,1733805904462 WAL splitting is done? wals=0, meta=false 2024-12-10T04:45:08,172 DEBUG [RS:2;6578523f4421:43039 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,43039,1733805907947 2024-12-10T04:45:08,173 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6578523f4421,43039,1733805907947] 2024-12-10T04:45:08,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733805905723, region=146881db9cca4e9e603d9b917ae4c321, ASSIGN}] 2024-12-10T04:45:08,179 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733805905723, region=146881db9cca4e9e603d9b917ae4c321, ASSIGN 2024-12-10T04:45:08,180 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733805905723, region=146881db9cca4e9e603d9b917ae4c321, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-10T04:45:08,186 INFO [RS:2;6578523f4421:43039 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T04:45:08,197 INFO [RS:2;6578523f4421:43039 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T04:45:08,201 INFO [RS:2;6578523f4421:43039 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T04:45:08,201 INFO [RS:2;6578523f4421:43039 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:08,209 INFO [RS:2;6578523f4421:43039 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T04:45:08,210 INFO [RS:2;6578523f4421:43039 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T04:45:08,210 INFO [RS:2;6578523f4421:43039 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:08,211 DEBUG [RS:2;6578523f4421:43039 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:08,211 DEBUG [RS:2;6578523f4421:43039 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:08,211 DEBUG [RS:2;6578523f4421:43039 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:08,211 DEBUG [RS:2;6578523f4421:43039 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:08,211 DEBUG [RS:2;6578523f4421:43039 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:08,212 DEBUG [RS:2;6578523f4421:43039 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6578523f4421:0, corePoolSize=2, maxPoolSize=2 2024-12-10T04:45:08,212 DEBUG [RS:2;6578523f4421:43039 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:08,212 DEBUG [RS:2;6578523f4421:43039 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:08,212 DEBUG [RS:2;6578523f4421:43039 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:08,212 DEBUG [RS:2;6578523f4421:43039 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:08,212 DEBUG [RS:2;6578523f4421:43039 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:08,212 DEBUG [RS:2;6578523f4421:43039 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6578523f4421:0, corePoolSize=1, maxPoolSize=1 2024-12-10T04:45:08,213 DEBUG [RS:2;6578523f4421:43039 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6578523f4421:0, corePoolSize=3, maxPoolSize=3 2024-12-10T04:45:08,213 DEBUG [RS:2;6578523f4421:43039 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6578523f4421:0, corePoolSize=3, maxPoolSize=3 2024-12-10T04:45:08,231 INFO [RS:2;6578523f4421:43039 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:08,232 INFO [RS:2;6578523f4421:43039 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:08,232 INFO [RS:2;6578523f4421:43039 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:08,232 INFO [RS:2;6578523f4421:43039 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:08,232 INFO [RS:2;6578523f4421:43039 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:08,232 INFO [RS:2;6578523f4421:43039 {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,43039,1733805907947-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T04:45:08,253 INFO [RS:2;6578523f4421:43039 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T04:45:08,253 INFO [RS:2;6578523f4421:43039 {}] hbase.ChoreService(168): Chore ScheduledChore name=6578523f4421,43039,1733805907947-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:08,254 INFO [RS:2;6578523f4421:43039 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:08,254 INFO [RS:2;6578523f4421:43039 {}] regionserver.Replication(171): 6578523f4421,43039,1733805907947 started 2024-12-10T04:45:08,272 INFO [RS:2;6578523f4421:43039 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T04:45:08,272 INFO [RS:2;6578523f4421:43039 {}] regionserver.HRegionServer(1482): Serving as 6578523f4421,43039,1733805907947, RpcServer on 6578523f4421/172.17.0.2:43039, sessionid=0x1000e0a982f0003 2024-12-10T04:45:08,272 DEBUG [RS:2;6578523f4421:43039 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T04:45:08,272 DEBUG [RS:2;6578523f4421:43039 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6578523f4421,43039,1733805907947 2024-12-10T04:45:08,272 DEBUG [RS:2;6578523f4421:43039 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6578523f4421,43039,1733805907947' 2024-12-10T04:45:08,272 DEBUG [RS:2;6578523f4421:43039 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T04:45:08,273 DEBUG [RS:2;6578523f4421:43039 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T04:45:08,274 DEBUG [RS:2;6578523f4421:43039 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T04:45:08,274 DEBUG [RS:2;6578523f4421:43039 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T04:45:08,274 DEBUG [RS:2;6578523f4421:43039 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6578523f4421,43039,1733805907947 2024-12-10T04:45:08,274 DEBUG [RS:2;6578523f4421:43039 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6578523f4421,43039,1733805907947' 2024-12-10T04:45:08,275 DEBUG [RS:2;6578523f4421:43039 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T04:45:08,275 DEBUG [RS:2;6578523f4421:43039 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T04:45:08,276 DEBUG [RS:2;6578523f4421:43039 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T04:45:08,276 INFO [RS:2;6578523f4421:43039 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T04:45:08,276 INFO [RS:2;6578523f4421:43039 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T04:45:08,331 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=146881db9cca4e9e603d9b917ae4c321, regionState=OPENING, regionLocation=6578523f4421,42295,1733805904421 2024-12-10T04:45:08,333 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6578523f4421:46207 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6578523f4421/172.17.0.2:46207 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T04:45:08,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6578523f4421,46207,1733805904462, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6578523f4421,46207,1733805904462, seqNum=-1, error=java.net.ConnectException: Call to address=6578523f4421:46207 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6578523f4421/172.17.0.2:46207 2024-12-10T04:45:08,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6578523f4421,46207,1733805904462, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-10T04:45:08,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6578523f4421,46207,1733805904462, seqNum=-1 from cache 2024-12-10T04:45:08,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 6578523f4421:46207 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6578523f4421/172.17.0.2:46207 2024-12-10T04:45:08,377 INFO [RS:2;6578523f4421:43039 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T04:45:08,378 INFO [RS:2;6578523f4421:43039 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6578523f4421%2C43039%2C1733805907947, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,43039,1733805907947, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/oldWALs, maxLogs=55 2024-12-10T04:45:08,408 INFO [RS:2;6578523f4421:43039 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/WALs/6578523f4421,43039,1733805907947/6578523f4421%2C43039%2C1733805907947.1733805908379 2024-12-10T04:45:08,408 DEBUG [RS:2;6578523f4421:43039 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T04:45:08,439 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T04:45:08,439 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6578523f4421,42295,1733805904421, seqNum=-1] 2024-12-10T04:45:08,439 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T04:45:08,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46661, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T04:45:08,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733805905723, region=146881db9cca4e9e603d9b917ae4c321, ASSIGN because future has completed 2024-12-10T04:45:08,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 146881db9cca4e9e603d9b917ae4c321, server=6578523f4421,42295,1733805904421}] 2024-12-10T04:45:08,600 INFO [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. 2024-12-10T04:45:08,600 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 146881db9cca4e9e603d9b917ae4c321, NAME => 'testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321.', STARTKEY => '', ENDKEY => ''} 2024-12-10T04:45:08,601 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733805905723 146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:08,601 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T04:45:08,602 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:08,602 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:08,603 INFO [StoreOpener-146881db9cca4e9e603d9b917ae4c321-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:08,604 INFO [StoreOpener-146881db9cca4e9e603d9b917ae4c321-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 146881db9cca4e9e603d9b917ae4c321 columnFamilyName family 2024-12-10T04:45:08,604 DEBUG [StoreOpener-146881db9cca4e9e603d9b917ae4c321-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T04:45:08,606 DEBUG [StoreFileOpener-146881db9cca4e9e603d9b917ae4c321-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321/family/8c32f883261b45f8a311a7e199b8dc1b, delay=923 ms 2024-12-10T04:45:08,608 DEBUG [StoreOpener-146881db9cca4e9e603d9b917ae4c321-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321/family/8c32f883261b45f8a311a7e199b8dc1b 2024-12-10T04:45:08,608 INFO [StoreOpener-146881db9cca4e9e603d9b917ae4c321-1 {}] regionserver.HStore(327): Store=146881db9cca4e9e603d9b917ae4c321/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T04:45:08,608 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:08,609 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:08,610 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:08,610 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:08,610 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:08,611 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:08,612 INFO [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 146881db9cca4e9e603d9b917ae4c321; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61451457, jitterRate=-0.08430193364620209}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T04:45:08,612 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:08,613 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 146881db9cca4e9e603d9b917ae4c321: Running coprocessor pre-open hook at 1733805908602Writing region info on filesystem at 1733805908602Initializing all the Stores at 1733805908602Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733805908602Cleaning up temporary data from old regions at 1733805908610 (+8 ms)Running coprocessor post-open hooks at 1733805908612 (+2 ms)Region opened successfully at 1733805908613 (+1 ms) 2024-12-10T04:45:08,614 INFO [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321., pid=11, masterSystemTime=1733805908598 2024-12-10T04:45:08,618 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=146881db9cca4e9e603d9b917ae4c321, regionState=OPEN, openSeqNum=10, regionLocation=6578523f4421,42295,1733805904421 2024-12-10T04:45:08,619 DEBUG [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. 2024-12-10T04:45:08,619 INFO [RS_OPEN_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. 2024-12-10T04:45:08,621 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 146881db9cca4e9e603d9b917ae4c321, server=6578523f4421,42295,1733805904421 because future has completed 2024-12-10T04:45:08,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-10T04:45:08,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 146881db9cca4e9e603d9b917ae4c321, server=6578523f4421,42295,1733805904421 in 177 msec 2024-12-10T04:45:08,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-12-10T04:45:08,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733805905723, region=146881db9cca4e9e603d9b917ae4c321, ASSIGN in 449 msec 2024-12-10T04:45:08,628 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server 6578523f4421,46207,1733805904462 after splitting done 2024-12-10T04:45:08,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 6578523f4421,46207,1733805904462, splitWal=true, meta=true in 940 msec 2024-12-10T04:45:08,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-10T04:45:08,772 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T04:45:08,772 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T04:45:08,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T04:45:08,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T04:45:08,773 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T04:45:08,773 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-10T04:45:08,773 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1090138123, stopped=false 2024-12-10T04:45:08,773 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=6578523f4421,42569,1733805904248 2024-12-10T04:45:08,823 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T04:45:08,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43039-0x1000e0a982f0003, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T04:45:08,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T04:45:08,823 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:08,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43039-0x1000e0a982f0003, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:08,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:08,824 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T04:45:08,824 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T04:45:08,824 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T04:45:08,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T04:45:08,824 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6578523f4421,42295,1733805904421' ***** 2024-12-10T04:45:08,824 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T04:45:08,824 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6578523f4421,43039,1733805907947' ***** 2024-12-10T04:45:08,824 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T04:45:08,825 INFO [RS:0;6578523f4421:42295 {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T04:45:08,825 INFO [RS:0;6578523f4421:42295 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630/bucket.persistence 2024-12-10T04:45:08,825 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T04:45:08,825 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T04:45:08,825 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T04:45:08,825 INFO [RS:0;6578523f4421:42295 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T04:45:08,825 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T04:45:08,827 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43039-0x1000e0a982f0003, quorum=127.0.0.1:57720, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T04:45:08,827 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T04:45:08,827 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T04:45:08,829 INFO [RS:2;6578523f4421:43039 {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T04:45:08,829 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T04:45:08,829 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T04:45:08,829 INFO [RS:2;6578523f4421:43039 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/420fe060-dc98-ec90-b4db-756b5ef53630/bucket.persistence 2024-12-10T04:45:08,829 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T04:45:08,829 INFO [RS:2;6578523f4421:43039 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T04:45:08,829 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T04:45:08,832 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T04:45:08,833 INFO [regionserver/6578523f4421:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T04:45:08,835 DEBUG [RS:0;6578523f4421:42295 {}] bucket.BucketCache(1487): Thread RS:0;6578523f4421:42295 started persisting bucket cache to file 2024-12-10T04:45:08,835 DEBUG [RS:0;6578523f4421:42295 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T04:45:08,835 DEBUG [RS:0;6578523f4421:42295 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T04:45:08,839 DEBUG [RS:0;6578523f4421:42295 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T04:45:08,839 DEBUG [RS:0;6578523f4421:42295 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733805908835 2024-12-10T04:45:08,839 DEBUG [RS:0;6578523f4421:42295 {}] bucket.BucketCache(1509): Thread RS:0;6578523f4421:42295 finished persisting bucket cache to file, renaming 2024-12-10T04:45:08,839 DEBUG [RS:2;6578523f4421:43039 {}] bucket.BucketCache(1487): Thread RS:2;6578523f4421:43039 started persisting bucket cache to file 2024-12-10T04:45:08,840 INFO [RS:0;6578523f4421:42295 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T04:45:08,840 DEBUG [RS:2;6578523f4421:43039 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T04:45:08,840 DEBUG [RS:2;6578523f4421:43039 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-12-10T04:45:08,840 INFO [RS:0;6578523f4421:42295 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T04:45:08,840 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T04:45:08,840 INFO [RS:0;6578523f4421:42295 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T04:45:08,840 INFO [RS:0;6578523f4421:42295 {}] regionserver.HRegionServer(3091): Received CLOSE for 146881db9cca4e9e603d9b917ae4c321 2024-12-10T04:45:08,840 INFO [RS:0;6578523f4421:42295 {}] regionserver.HRegionServer(959): stopping server 6578523f4421,42295,1733805904421 2024-12-10T04:45:08,840 INFO [RS:0;6578523f4421:42295 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T04:45:08,840 INFO [RS:0;6578523f4421:42295 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;6578523f4421:42295. 2024-12-10T04:45:08,840 INFO [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T04:45:08,840 DEBUG [RS:0;6578523f4421:42295 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T04:45:08,841 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 146881db9cca4e9e603d9b917ae4c321, disabling compactions & flushes 2024-12-10T04:45:08,841 DEBUG [RS:0;6578523f4421:42295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T04:45:08,841 INFO [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. 2024-12-10T04:45:08,841 INFO [RS:0;6578523f4421:42295 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T04:45:08,841 INFO [RS:0;6578523f4421:42295 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T04:45:08,841 INFO [RS:0;6578523f4421:42295 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T04:45:08,841 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. 2024-12-10T04:45:08,841 INFO [RS:0;6578523f4421:42295 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-10T04:45:08,841 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. after waiting 0 ms 2024-12-10T04:45:08,841 INFO [RS:0;6578523f4421:42295 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-10T04:45:08,841 DEBUG [RS:0;6578523f4421:42295 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 146881db9cca4e9e603d9b917ae4c321=testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321.} 2024-12-10T04:45:08,841 DEBUG [RS:0;6578523f4421:42295 {}] regionserver.HRegionServer(1351): Waiting on 146881db9cca4e9e603d9b917ae4c321, 1588230740 2024-12-10T04:45:08,841 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T04:45:08,842 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T04:45:08,842 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T04:45:08,841 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. 2024-12-10T04:45:08,842 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T04:45:08,842 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T04:45:08,842 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T04:45:08,842 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-12-10T04:45:08,844 DEBUG [RS:2;6578523f4421:43039 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-12-10T04:45:08,844 DEBUG [RS:2;6578523f4421:43039 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1733805908839 2024-12-10T04:45:08,844 DEBUG [RS:2;6578523f4421:43039 {}] bucket.BucketCache(1509): Thread RS:2;6578523f4421:43039 finished persisting bucket cache to file, renaming 2024-12-10T04:45:08,844 INFO [RS:2;6578523f4421:43039 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T04:45:08,845 INFO [RS:2;6578523f4421:43039 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T04:45:08,845 INFO [RS:2;6578523f4421:43039 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T04:45:08,845 DEBUG [StoreFileCloser-146881db9cca4e9e603d9b917ae4c321-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321/family/8c32f883261b45f8a311a7e199b8dc1b 2024-12-10T04:45:08,845 INFO [RS:2;6578523f4421:43039 {}] regionserver.HRegionServer(959): stopping server 6578523f4421,43039,1733805907947 2024-12-10T04:45:08,845 INFO [RS:2;6578523f4421:43039 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T04:45:08,845 INFO [RS:2;6578523f4421:43039 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;6578523f4421:43039. 2024-12-10T04:45:08,845 DEBUG [RS:2;6578523f4421:43039 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T04:45:08,845 DEBUG [RS:2;6578523f4421:43039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T04:45:08,845 INFO [RS:2;6578523f4421:43039 {}] regionserver.HRegionServer(976): stopping server 6578523f4421,43039,1733805907947; all regions closed. 2024-12-10T04:45:08,850 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/default/testBlockEvictionOnGracefulStop1733805905723/146881db9cca4e9e603d9b917ae4c321/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-10T04:45:08,851 INFO [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. 2024-12-10T04:45:08,851 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 146881db9cca4e9e603d9b917ae4c321: Waiting for close lock at 1733805908840Running coprocessor pre-close hooks at 1733805908840Disabling compacts and flushes for region at 1733805908840Disabling writes for close at 1733805908841 (+1 ms)Writing region close event to WAL at 1733805908845 (+4 ms)Running coprocessor post-close hooks at 1733805908851 (+6 ms)Closed at 1733805908851 2024-12-10T04:45:08,851 DEBUG [RS_CLOSE_REGION-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321. 2024-12-10T04:45:08,852 DEBUG [RS:2;6578523f4421:43039 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/oldWALs 2024-12-10T04:45:08,852 INFO [RS:2;6578523f4421:43039 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6578523f4421%2C43039%2C1733805907947:(num 1733805908379) 2024-12-10T04:45:08,852 DEBUG [RS:2;6578523f4421:43039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T04:45:08,852 INFO [RS:2;6578523f4421:43039 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T04:45:08,852 INFO [RS:2;6578523f4421:43039 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T04:45:08,852 INFO [RS:2;6578523f4421:43039 {}] hbase.ChoreService(370): Chore service for: regionserver/6578523f4421:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T04:45:08,852 INFO [RS:2;6578523f4421:43039 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T04:45:08,852 INFO [regionserver/6578523f4421:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T04:45:08,852 INFO [RS:2;6578523f4421:43039 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T04:45:08,852 INFO [RS:2;6578523f4421:43039 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T04:45:08,853 INFO [RS:2;6578523f4421:43039 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T04:45:08,853 INFO [RS:2;6578523f4421:43039 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43039 2024-12-10T04:45:08,859 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a0feb732424c434581c2059f87b94f3e 2024-12-10T04:45:08,875 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/info/a0feb732424c434581c2059f87b94f3e is 209, key is testBlockEvictionOnGracefulStop1733805905723,,1733805905731.146881db9cca4e9e603d9b917ae4c321./info:regioninfo/1733805908617/Put/seqid=0 2024-12-10T04:45:08,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43039-0x1000e0a982f0003, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6578523f4421,43039,1733805907947 2024-12-10T04:45:08,876 INFO [RS:2;6578523f4421:43039 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T04:45:08,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T04:45:08,876 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/info/a0feb732424c434581c2059f87b94f3e 2024-12-10T04:45:08,876 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$249/0x00007f5548477b20@57a2be5 rejected from java.util.concurrent.ThreadPoolExecutor@4078a52e[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 4] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-10T04:45:08,879 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/.tmp/info/a0feb732424c434581c2059f87b94f3e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/info/a0feb732424c434581c2059f87b94f3e 2024-12-10T04:45:08,881 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/info/a0feb732424c434581c2059f87b94f3e, delay=907 ms 2024-12-10T04:45:08,882 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/info/a0feb732424c434581c2059f87b94f3e, entries=8, sequenceid=19, filesize=6.6 K 2024-12-10T04:45:08,883 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 41ms, sequenceid=19, compaction requested=false 2024-12-10T04:45:08,884 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/info/6acfe5b9dd914a45aa8b4caa2fa3b84d 2024-12-10T04:45:08,885 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/info/a0feb732424c434581c2059f87b94f3e 2024-12-10T04:45:08,886 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6578523f4421,43039,1733805907947] 2024-12-10T04:45:08,889 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/ns/3e5b7752e9bd448397a2240c3eb8a7e9 2024-12-10T04:45:08,890 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/table/761c64c62e4740069a8905809e21b091 2024-12-10T04:45:08,895 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-12-10T04:45:08,896 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T04:45:08,896 INFO [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T04:45:08,896 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733805908841Running coprocessor pre-close hooks at 1733805908841Disabling compacts and flushes for region at 1733805908841Disabling writes for close at 1733805908842 (+1 ms)Obtaining lock to block concurrent updates at 1733805908842Preparing flush snapshotting stores in 1588230740 at 1733805908842Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1733805908843 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733805908843Flushing 1588230740/info: creating writer at 1733805908843Flushing 1588230740/info: appending metadata at 1733805908875 (+32 ms)Flushing 1588230740/info: closing flushed file at 1733805908875Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45f6d3f2: reopening flushed file at 1733805908879 (+4 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 41ms, sequenceid=19, compaction requested=false at 1733805908883 (+4 ms)Writing region close event to WAL at 1733805908891 (+8 ms)Running coprocessor post-close hooks at 1733805908895 (+4 ms)Closed at 1733805908896 (+1 ms) 2024-12-10T04:45:08,896 DEBUG [RS_CLOSE_META-regionserver/6578523f4421:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T04:45:08,896 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6578523f4421,43039,1733805907947 already deleted, retry=false 2024-12-10T04:45:08,897 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6578523f4421,43039,1733805907947 expired; onlineServers=1 2024-12-10T04:45:08,923 INFO [regionserver/6578523f4421:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T04:45:08,986 INFO [RS:2;6578523f4421:43039 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T04:45:08,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43039-0x1000e0a982f0003, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T04:45:08,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43039-0x1000e0a982f0003, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T04:45:08,986 INFO [RS:2;6578523f4421:43039 {}] regionserver.HRegionServer(1031): Exiting; stopping=6578523f4421,43039,1733805907947; zookeeper connection closed. 2024-12-10T04:45:08,987 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@69fa5cfb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@69fa5cfb 2024-12-10T04:45:09,042 INFO [RS:0;6578523f4421:42295 {}] regionserver.HRegionServer(976): stopping server 6578523f4421,42295,1733805904421; all regions closed. 2024-12-10T04:45:09,044 DEBUG [RS:0;6578523f4421:42295 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/oldWALs 2024-12-10T04:45:09,044 INFO [RS:0;6578523f4421:42295 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6578523f4421%2C42295%2C1733805904421.meta:.meta(num 1733805908071) 2024-12-10T04:45:09,046 DEBUG [RS:0;6578523f4421:42295 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/oldWALs 2024-12-10T04:45:09,046 INFO [RS:0;6578523f4421:42295 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6578523f4421%2C42295%2C1733805904421:(num 1733805905155) 2024-12-10T04:45:09,046 DEBUG [RS:0;6578523f4421:42295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T04:45:09,046 INFO [RS:0;6578523f4421:42295 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T04:45:09,046 INFO [RS:0;6578523f4421:42295 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T04:45:09,046 INFO [RS:0;6578523f4421:42295 {}] hbase.ChoreService(370): Chore service for: regionserver/6578523f4421:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T04:45:09,047 INFO [RS:0;6578523f4421:42295 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T04:45:09,047 INFO [regionserver/6578523f4421:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T04:45:09,047 INFO [RS:0;6578523f4421:42295 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42295 2024-12-10T04:45:09,055 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6578523f4421,42295,1733805904421 2024-12-10T04:45:09,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T04:45:09,055 INFO [RS:0;6578523f4421:42295 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T04:45:09,056 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6578523f4421,42295,1733805904421] 2024-12-10T04:45:09,075 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6578523f4421,42295,1733805904421 already deleted, retry=false 2024-12-10T04:45:09,076 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6578523f4421,42295,1733805904421 expired; onlineServers=0 2024-12-10T04:45:09,076 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '6578523f4421,42569,1733805904248' ***** 2024-12-10T04:45:09,076 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-10T04:45:09,076 INFO [M:0;6578523f4421:42569 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T04:45:09,076 INFO [M:0;6578523f4421:42569 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T04:45:09,076 DEBUG [M:0;6578523f4421:42569 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-10T04:45:09,076 DEBUG [M:0;6578523f4421:42569 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-10T04:45:09,076 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-10T04:45:09,076 DEBUG [master/6578523f4421:0:becomeActiveMaster-HFileCleaner.large.0-1733805904888 {}] cleaner.HFileCleaner(306): Exit Thread[master/6578523f4421:0:becomeActiveMaster-HFileCleaner.large.0-1733805904888,5,FailOnTimeoutGroup] 2024-12-10T04:45:09,076 DEBUG [master/6578523f4421:0:becomeActiveMaster-HFileCleaner.small.0-1733805904888 {}] cleaner.HFileCleaner(306): Exit Thread[master/6578523f4421:0:becomeActiveMaster-HFileCleaner.small.0-1733805904888,5,FailOnTimeoutGroup] 2024-12-10T04:45:09,077 INFO [M:0;6578523f4421:42569 {}] hbase.ChoreService(370): Chore service for: master/6578523f4421:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-10T04:45:09,077 INFO [M:0;6578523f4421:42569 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T04:45:09,077 DEBUG [M:0;6578523f4421:42569 {}] master.HMaster(1795): Stopping service threads 2024-12-10T04:45:09,077 INFO [M:0;6578523f4421:42569 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-10T04:45:09,077 INFO [M:0;6578523f4421:42569 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T04:45:09,077 INFO [M:0;6578523f4421:42569 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-10T04:45:09,077 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-10T04:45:09,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-10T04:45:09,086 DEBUG [M:0;6578523f4421:42569 {}] zookeeper.ZKUtil(347): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-10T04:45:09,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T04:45:09,086 WARN [M:0;6578523f4421:42569 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-10T04:45:09,087 INFO [M:0;6578523f4421:42569 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/.lastflushedseqids 2024-12-10T04:45:09,093 INFO [M:0;6578523f4421:42569 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-10T04:45:09,093 INFO [M:0;6578523f4421:42569 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-10T04:45:09,093 DEBUG [M:0;6578523f4421:42569 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T04:45:09,093 INFO [M:0;6578523f4421:42569 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T04:45:09,093 DEBUG [M:0;6578523f4421:42569 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T04:45:09,093 DEBUG [M:0;6578523f4421:42569 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T04:45:09,093 DEBUG [M:0;6578523f4421:42569 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T04:45:09,094 INFO [M:0;6578523f4421:42569 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-12-10T04:45:09,104 DEBUG [M:0;6578523f4421:42569 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 95ed4cea3887422793e20a038d2333be 2024-12-10T04:45:09,120 DEBUG [M:0;6578523f4421:42569 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/95ed4cea3887422793e20a038d2333be is 82, key is hbase:meta,,1/info:regioninfo/1733805908153/Put/seqid=0 2024-12-10T04:45:09,121 INFO [M:0;6578523f4421:42569 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/95ed4cea3887422793e20a038d2333be 2024-12-10T04:45:09,129 DEBUG [M:0;6578523f4421:42569 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 468b756811db4e99b693b89a2dbee8a9 2024-12-10T04:45:09,145 DEBUG [M:0;6578523f4421:42569 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/468b756811db4e99b693b89a2dbee8a9 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733805906131/Put/seqid=0 2024-12-10T04:45:09,146 INFO [M:0;6578523f4421:42569 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/468b756811db4e99b693b89a2dbee8a9 2024-12-10T04:45:09,154 DEBUG [M:0;6578523f4421:42569 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f6237dea20214737bba166f580dd09a2 2024-12-10T04:45:09,165 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T04:45:09,165 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42295-0x1000e0a982f0001, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T04:45:09,165 INFO [RS:0;6578523f4421:42295 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T04:45:09,166 INFO [RS:0;6578523f4421:42295 {}] regionserver.HRegionServer(1031): Exiting; stopping=6578523f4421,42295,1733805904421; zookeeper connection closed. 2024-12-10T04:45:09,166 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@afaee35 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@afaee35 2024-12-10T04:45:09,166 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-10T04:45:09,170 DEBUG [M:0;6578523f4421:42569 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f6237dea20214737bba166f580dd09a2 is 69, key is 6578523f4421,42295,1733805904421/rs:state/1733805904940/Put/seqid=0 2024-12-10T04:45:09,171 INFO [M:0;6578523f4421:42569 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f6237dea20214737bba166f580dd09a2 2024-12-10T04:45:09,173 INFO [M:0;6578523f4421:42569 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f6237dea20214737bba166f580dd09a2 2024-12-10T04:45:09,174 DEBUG [M:0;6578523f4421:42569 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/95ed4cea3887422793e20a038d2333be as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/95ed4cea3887422793e20a038d2333be 2024-12-10T04:45:09,176 INFO [M:0;6578523f4421:42569 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/95ed4cea3887422793e20a038d2333be, entries=14, sequenceid=102, filesize=5.9 K 2024-12-10T04:45:09,176 DEBUG [M:0;6578523f4421:42569 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/468b756811db4e99b693b89a2dbee8a9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/468b756811db4e99b693b89a2dbee8a9 2024-12-10T04:45:09,178 INFO [M:0;6578523f4421:42569 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/468b756811db4e99b693b89a2dbee8a9, entries=11, sequenceid=102, filesize=6.5 K 2024-12-10T04:45:09,179 DEBUG [M:0;6578523f4421:42569 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f6237dea20214737bba166f580dd09a2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f6237dea20214737bba166f580dd09a2 2024-12-10T04:45:09,181 INFO [M:0;6578523f4421:42569 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f6237dea20214737bba166f580dd09a2 2024-12-10T04:45:09,181 INFO [M:0;6578523f4421:42569 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a3b79cb4-075a-caed-7b17-2ed8fa3563e2/0b5b1f4d-b746-edbc-0295-dddee85d1570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f6237dea20214737bba166f580dd09a2, entries=3, sequenceid=102, filesize=5.3 K 2024-12-10T04:45:09,182 INFO [M:0;6578523f4421:42569 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 89ms, sequenceid=102, compaction requested=false 2024-12-10T04:45:09,185 INFO [M:0;6578523f4421:42569 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T04:45:09,185 DEBUG [M:0;6578523f4421:42569 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733805909093Disabling compacts and flushes for region at 1733805909093Disabling writes for close at 1733805909093Obtaining lock to block concurrent updates at 1733805909094 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733805909094Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1733805909094Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733805909094Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733805909095 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733805909119 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733805909120 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733805909123 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733805909144 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733805909144Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733805909147 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733805909170 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733805909170Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66cae6c4: reopening flushed file at 1733805909173 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d353f24: reopening flushed file at 1733805909176 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@454a0d2c: reopening flushed file at 1733805909178 (+2 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 89ms, sequenceid=102, compaction requested=false at 1733805909182 (+4 ms)Writing region close event to WAL at 1733805909184 (+2 ms)Closed at 1733805909184 2024-12-10T04:45:09,186 INFO [M:0;6578523f4421:42569 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-10T04:45:09,186 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T04:45:09,186 INFO [M:0;6578523f4421:42569 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42569 2024-12-10T04:45:09,187 INFO [M:0;6578523f4421:42569 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T04:45:09,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T04:45:09,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42569-0x1000e0a982f0000, quorum=127.0.0.1:57720, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T04:45:09,300 INFO [M:0;6578523f4421:42569 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T04:45:09,303 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-10T04:45:09,304 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-10T04:45:09,304 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T04:45:09,311 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=275 (was 300), ProcessCount=11 (was 11), AvailableMemoryMB=5525 (was 5647)