2024-12-01 14:55:09,703 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-01 14:55:09,746 main DEBUG Took 0.039506 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-01 14:55:09,747 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-01 14:55:09,747 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-01 14:55:09,749 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-01 14:55:09,751 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:55:09,759 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-01 14:55:09,773 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:55:09,774 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:55:09,775 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:55:09,775 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:55:09,776 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:55:09,776 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:55:09,777 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:55:09,777 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:55:09,778 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:55:09,778 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:55:09,779 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:55:09,779 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:55:09,780 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:55:09,782 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:55:09,782 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:55:09,783 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:55:09,784 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:55:09,784 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:55:09,785 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:55:09,785 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:55:09,786 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:55:09,787 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:55:09,787 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:55:09,788 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:55:09,788 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:55:09,789 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-01 14:55:09,790 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:55:09,792 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-01 14:55:09,794 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-01 14:55:09,795 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-01 14:55:09,797 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-01 14:55:09,798 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-01 14:55:09,809 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-01 14:55:09,815 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-01 14:55:09,817 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-01 14:55:09,818 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-01 14:55:09,818 main DEBUG createAppenders(={Console}) 2024-12-01 14:55:09,819 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-01 14:55:09,820 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-01 14:55:09,820 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-01 14:55:09,821 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-01 14:55:09,821 main DEBUG OutputStream closed 2024-12-01 14:55:09,821 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-01 14:55:09,822 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-01 14:55:09,822 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-01 14:55:09,903 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-01 14:55:09,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-01 14:55:09,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-01 14:55:09,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-01 14:55:09,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-01 14:55:09,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-01 14:55:09,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-01 14:55:09,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-01 14:55:09,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-01 14:55:09,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-01 14:55:09,910 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-01 14:55:09,910 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-01 14:55:09,911 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-01 14:55:09,911 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-01 14:55:09,912 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-01 14:55:09,912 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-01 14:55:09,912 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-01 14:55:09,913 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-01 14:55:09,916 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-01 14:55:09,916 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-01 14:55:09,917 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-01 14:55:09,918 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-01T14:55:09,934 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-01 14:55:09,937 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-01 14:55:09,937 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-01T14:55:10,251 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965 2024-12-01T14:55:10,280 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=381, ProcessCount=11, AvailableMemoryMB=5520 2024-12-01T14:55:10,388 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-01T14:55:10,507 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:10,527 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/cluster_8af70a8e-e4d2-e15a-d6f9-62373f81eea3, deleteOnExit=true 2024-12-01T14:55:10,693 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/cluster_8af70a8e-e4d2-e15a-d6f9-62373f81eea3/zookeeper_0, clientPort=58374, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/cluster_8af70a8e-e4d2-e15a-d6f9-62373f81eea3/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/cluster_8af70a8e-e4d2-e15a-d6f9-62373f81eea3/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-01T14:55:10,708 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58374 2024-12-01T14:55:10,709 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:10,808 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46 with version=8 2024-12-01T14:55:10,809 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/hbase-staging 2024-12-01T14:55:10,915 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-01T14:55:11,194 INFO [Time-limited test {}] client.ConnectionUtils(128): master/519b220a2ba1:0 server-side Connection retries=45 2024-12-01T14:55:11,206 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:11,208 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:11,219 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T14:55:11,219 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:11,220 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T14:55:11,360 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-01T14:55:11,451 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T14:55:11,504 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-01T14:55:11,519 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-01T14:55:11,524 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T14:55:11,569 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 107930 (auto-detected) 2024-12-01T14:55:11,570 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-01T14:55:11,589 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33719 2024-12-01T14:55:11,613 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33719 connecting to ZooKeeper ensemble=127.0.0.1:58374 2024-12-01T14:55:11,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:337190x0, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T14:55:11,730 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33719-0x10192ad7a110000 connected 2024-12-01T14:55:11,810 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:11,811 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:11,821 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T14:55:11,825 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46, hbase.cluster.distributed=false 2024-12-01T14:55:11,852 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T14:55:11,859 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33719 2024-12-01T14:55:11,860 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33719 2024-12-01T14:55:11,860 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33719 2024-12-01T14:55:11,861 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33719 2024-12-01T14:55:11,861 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33719 2024-12-01T14:55:11,950 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/519b220a2ba1:0 server-side Connection retries=45 2024-12-01T14:55:11,951 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:11,952 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:11,952 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T14:55:11,952 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:11,952 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T14:55:11,956 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T14:55:11,959 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T14:55:11,961 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43229 2024-12-01T14:55:11,963 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43229 connecting to ZooKeeper ensemble=127.0.0.1:58374 2024-12-01T14:55:11,963 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:11,964 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:11,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:432290x0, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T14:55:11,990 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:432290x0, quorum=127.0.0.1:58374, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T14:55:11,990 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43229-0x10192ad7a110001 connected 2024-12-01T14:55:11,996 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T14:55:12,008 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/bucket.cache 2024-12-01T14:55:12,014 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-01T14:55:12,015 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-01T14:55:12,017 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-01T14:55:12,019 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-01T14:55:12,019 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-01T14:55:12,026 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-01T14:55:12,029 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-01T14:55:12,030 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/bucket.persistence is complete. 2024-12-01T14:55:12,038 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T14:55:12,043 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43229-0x10192ad7a110001, quorum=127.0.0.1:58374, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-01T14:55:12,049 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43229-0x10192ad7a110001, quorum=127.0.0.1:58374, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T14:55:12,055 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43229 2024-12-01T14:55:12,058 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43229 2024-12-01T14:55:12,059 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43229 2024-12-01T14:55:12,063 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43229 2024-12-01T14:55:12,063 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43229 2024-12-01T14:55:12,090 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/519b220a2ba1:0 server-side Connection retries=45 2024-12-01T14:55:12,090 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:12,091 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:12,091 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T14:55:12,091 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:12,092 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T14:55:12,092 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T14:55:12,092 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T14:55:12,094 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42799 2024-12-01T14:55:12,097 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42799 connecting to ZooKeeper ensemble=127.0.0.1:58374 2024-12-01T14:55:12,097 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:12,098 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:12,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:427990x0, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T14:55:12,132 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:427990x0, quorum=127.0.0.1:58374, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T14:55:12,133 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42799-0x10192ad7a110002 connected 2024-12-01T14:55:12,133 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T14:55:12,134 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/bucket.cache 2024-12-01T14:55:12,134 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-01T14:55:12,135 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-01T14:55:12,135 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-01T14:55:12,136 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-01T14:55:12,137 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-01T14:55:12,137 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-01T14:55:12,138 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-01T14:55:12,140 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T14:55:12,142 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42799-0x10192ad7a110002, quorum=127.0.0.1:58374, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-01T14:55:12,140 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/bucket.persistence is complete. 2024-12-01T14:55:12,144 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42799-0x10192ad7a110002, quorum=127.0.0.1:58374, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T14:55:12,146 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42799 2024-12-01T14:55:12,148 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42799 2024-12-01T14:55:12,154 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42799 2024-12-01T14:55:12,159 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42799 2024-12-01T14:55:12,167 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42799 2024-12-01T14:55:12,190 DEBUG [M:0;519b220a2ba1:33719 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;519b220a2ba1:33719 2024-12-01T14:55:12,192 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/519b220a2ba1,33719,1733064910998 2024-12-01T14:55:12,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43229-0x10192ad7a110001, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T14:55:12,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42799-0x10192ad7a110002, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T14:55:12,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T14:55:12,222 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/519b220a2ba1,33719,1733064910998 2024-12-01T14:55:12,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43229-0x10192ad7a110001, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-01T14:55:12,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42799-0x10192ad7a110002, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-01T14:55:12,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42799-0x10192ad7a110002, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:12,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43229-0x10192ad7a110001, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:12,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:12,269 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-01T14:55:12,270 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/519b220a2ba1,33719,1733064910998 from backup master directory 2024-12-01T14:55:12,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/519b220a2ba1,33719,1733064910998 2024-12-01T14:55:12,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43229-0x10192ad7a110001, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T14:55:12,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42799-0x10192ad7a110002, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T14:55:12,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T14:55:12,282 WARN [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T14:55:12,282 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=519b220a2ba1,33719,1733064910998 2024-12-01T14:55:12,286 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-01T14:55:12,288 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-01T14:55:12,301 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/hbase.id] with ID: 010b4207-fc35-4102-a56d-e2ac761de6a9 2024-12-01T14:55:12,301 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/.tmp/hbase.id 2024-12-01T14:55:12,322 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/hbase.id] 2024-12-01T14:55:12,422 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:12,427 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-01T14:55:12,453 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-12-01T14:55:12,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43229-0x10192ad7a110001, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:12,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42799-0x10192ad7a110002, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:12,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:12,545 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T14:55:12,553 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-01T14:55:12,618 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:55:12,626 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:55:12,727 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store 2024-12-01T14:55:12,772 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-01T14:55:12,776 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:55:12,780 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-01T14:55:12,780 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:55:12,781 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:55:12,787 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 1 ms 2024-12-01T14:55:12,787 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:55:12,788 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:55:12,789 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733064912780Disabling compacts and flushes for region at 1733064912780Disabling writes for close at 1733064912787 (+7 ms)Writing region close event to WAL at 1733064912788 (+1 ms)Closed at 1733064912788 2024-12-01T14:55:12,789 WARN [master/519b220a2ba1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/.initializing 2024-12-01T14:55:12,790 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/WALs/519b220a2ba1,33719,1733064910998 2024-12-01T14:55:12,821 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:55:12,838 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=519b220a2ba1%2C33719%2C1733064910998, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/WALs/519b220a2ba1,33719,1733064910998, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/oldWALs, maxLogs=10 2024-12-01T14:55:12,888 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/WALs/519b220a2ba1,33719,1733064910998/519b220a2ba1%2C33719%2C1733064910998.1733064912843 2024-12-01T14:55:12,892 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-01T14:55:12,893 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:55:12,893 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:55:12,897 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:12,899 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:12,949 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:12,997 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-01T14:55:13,004 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:13,008 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:13,009 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:13,045 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-01T14:55:13,046 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:13,048 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:55:13,049 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:13,059 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-01T14:55:13,060 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:13,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:55:13,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:13,090 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-01T14:55:13,091 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:13,092 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:55:13,092 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:13,096 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:13,097 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:13,101 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:13,102 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:13,109 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T14:55:13,115 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:13,137 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T14:55:13,138 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62637609, jitterRate=-0.0666268914937973}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T14:55:13,148 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733064912914Initializing all the Stores at 1733064912915 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733064912916 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733064912918 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733064912921 (+3 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733064912921Cleaning up temporary data from old regions at 1733064913102 (+181 ms)Region opened successfully at 1733064913147 (+45 ms) 2024-12-01T14:55:13,150 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-01T14:55:13,209 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eadc481, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=519b220a2ba1/172.17.0.2:0 2024-12-01T14:55:13,256 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-01T14:55:13,270 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-01T14:55:13,270 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-01T14:55:13,275 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-01T14:55:13,276 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-01T14:55:13,289 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-12-01T14:55:13,289 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-01T14:55:13,329 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-01T14:55:13,343 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-01T14:55:13,372 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-01T14:55:13,379 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-01T14:55:13,381 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-01T14:55:13,388 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-01T14:55:13,391 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-01T14:55:13,397 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-01T14:55:13,411 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-01T14:55:13,412 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-01T14:55:13,422 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-01T14:55:13,445 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-01T14:55:13,455 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-01T14:55:13,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T14:55:13,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:13,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43229-0x10192ad7a110001, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T14:55:13,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43229-0x10192ad7a110001, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:13,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42799-0x10192ad7a110002, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T14:55:13,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42799-0x10192ad7a110002, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:13,487 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=519b220a2ba1,33719,1733064910998, sessionid=0x10192ad7a110000, setting cluster-up flag (Was=false) 2024-12-01T14:55:13,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42799-0x10192ad7a110002, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:13,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43229-0x10192ad7a110001, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:13,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:13,547 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-01T14:55:13,556 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=519b220a2ba1,33719,1733064910998 2024-12-01T14:55:13,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43229-0x10192ad7a110001, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:13,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:13,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42799-0x10192ad7a110002, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:13,605 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-01T14:55:13,607 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=519b220a2ba1,33719,1733064910998 2024-12-01T14:55:13,612 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-01T14:55:13,676 INFO [RS:0;519b220a2ba1:43229 {}] regionserver.HRegionServer(746): ClusterId : 010b4207-fc35-4102-a56d-e2ac761de6a9 2024-12-01T14:55:13,677 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.HRegionServer(746): ClusterId : 010b4207-fc35-4102-a56d-e2ac761de6a9 2024-12-01T14:55:13,678 DEBUG [RS:0;519b220a2ba1:43229 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T14:55:13,678 DEBUG [RS:1;519b220a2ba1:42799 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T14:55:13,687 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-01T14:55:13,697 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-01T14:55:13,705 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-01T14:55:13,711 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 519b220a2ba1,33719,1733064910998 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-01T14:55:13,723 DEBUG [RS:0;519b220a2ba1:43229 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T14:55:13,723 DEBUG [RS:0;519b220a2ba1:43229 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T14:55:13,724 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/519b220a2ba1:0, corePoolSize=5, maxPoolSize=5 2024-12-01T14:55:13,724 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/519b220a2ba1:0, corePoolSize=5, maxPoolSize=5 2024-12-01T14:55:13,724 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/519b220a2ba1:0, corePoolSize=5, maxPoolSize=5 2024-12-01T14:55:13,724 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/519b220a2ba1:0, corePoolSize=5, maxPoolSize=5 2024-12-01T14:55:13,725 DEBUG [RS:1;519b220a2ba1:42799 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T14:55:13,725 DEBUG [RS:1;519b220a2ba1:42799 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T14:55:13,725 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/519b220a2ba1:0, corePoolSize=10, maxPoolSize=10 2024-12-01T14:55:13,725 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:13,725 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/519b220a2ba1:0, corePoolSize=2, maxPoolSize=2 2024-12-01T14:55:13,725 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:13,731 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T14:55:13,732 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-01T14:55:13,738 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:13,738 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-01T14:55:13,740 DEBUG [RS:1;519b220a2ba1:42799 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T14:55:13,741 DEBUG [RS:1;519b220a2ba1:42799 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7054f1d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=519b220a2ba1/172.17.0.2:0 2024-12-01T14:55:13,744 DEBUG [RS:0;519b220a2ba1:43229 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T14:55:13,745 DEBUG [RS:0;519b220a2ba1:43229 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3304b401, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=519b220a2ba1/172.17.0.2:0 2024-12-01T14:55:13,747 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733064943747 2024-12-01T14:55:13,749 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-01T14:55:13,750 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-01T14:55:13,753 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-01T14:55:13,753 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-01T14:55:13,754 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-01T14:55:13,754 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-01T14:55:13,755 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:13,774 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-01T14:55:13,776 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-01T14:55:13,777 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-01T14:55:13,777 DEBUG [RS:1;519b220a2ba1:42799 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;519b220a2ba1:42799 2024-12-01T14:55:13,778 DEBUG [RS:0;519b220a2ba1:43229 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;519b220a2ba1:43229 2024-12-01T14:55:13,779 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-01T14:55:13,780 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-01T14:55:13,784 INFO [RS:0;519b220a2ba1:43229 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T14:55:13,784 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T14:55:13,784 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T14:55:13,784 INFO [RS:0;519b220a2ba1:43229 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T14:55:13,784 DEBUG [RS:1;519b220a2ba1:42799 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T14:55:13,784 DEBUG [RS:0;519b220a2ba1:43229 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T14:55:13,787 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.HRegionServer(2659): reportForDuty to master=519b220a2ba1,33719,1733064910998 with port=42799, startcode=1733064912089 2024-12-01T14:55:13,788 INFO [RS:0;519b220a2ba1:43229 {}] regionserver.HRegionServer(2659): reportForDuty to master=519b220a2ba1,33719,1733064910998 with port=43229, startcode=1733064911911 2024-12-01T14:55:13,798 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/519b220a2ba1:0:becomeActiveMaster-HFileCleaner.large.0-1733064913783,5,FailOnTimeoutGroup] 2024-12-01T14:55:13,803 DEBUG [RS:1;519b220a2ba1:42799 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T14:55:13,803 DEBUG [RS:0;519b220a2ba1:43229 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T14:55:13,808 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/519b220a2ba1:0:becomeActiveMaster-HFileCleaner.small.0-1733064913799,5,FailOnTimeoutGroup] 2024-12-01T14:55:13,808 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:13,809 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-01T14:55:13,810 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:13,811 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:13,812 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-01T14:55:13,812 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46 2024-12-01T14:55:13,828 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:55:13,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-01T14:55:13,869 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T14:55:13,870 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:13,870 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:13,874 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-01T14:55:13,900 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T14:55:13,900 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:13,900 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:13,901 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-01T14:55:13,901 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47315, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T14:55:13,902 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58947, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T14:55:13,909 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33719 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 519b220a2ba1,42799,1733064912089 2024-12-01T14:55:13,916 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33719 {}] master.ServerManager(517): Registering regionserver=519b220a2ba1,42799,1733064912089 2024-12-01T14:55:13,921 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T14:55:13,921 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:13,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:13,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-01T14:55:13,931 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33719 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 519b220a2ba1,43229,1733064911911 2024-12-01T14:55:13,931 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33719 {}] master.ServerManager(517): Registering regionserver=519b220a2ba1,43229,1733064911911 2024-12-01T14:55:13,936 DEBUG [RS:1;519b220a2ba1:42799 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46 2024-12-01T14:55:13,936 DEBUG [RS:0;519b220a2ba1:43229 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46 2024-12-01T14:55:13,936 DEBUG [RS:1;519b220a2ba1:42799 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-01T14:55:13,936 DEBUG [RS:1;519b220a2ba1:42799 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-01T14:55:13,938 DEBUG [RS:0;519b220a2ba1:43229 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-01T14:55:13,938 DEBUG [RS:0;519b220a2ba1:43229 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-01T14:55:13,947 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T14:55:13,947 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:13,948 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:13,948 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T14:55:13,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740 2024-12-01T14:55:13,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740 2024-12-01T14:55:13,951 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T14:55:13,951 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T14:55:13,951 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T14:55:13,952 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T14:55:13,971 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T14:55:13,972 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74540561, jitterRate=0.11074091494083405}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T14:55:13,976 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733064913829Initializing all the Stores at 1733064913829Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733064913829Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733064913834 (+5 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733064913837 (+3 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733064913837Cleaning up temporary data from old regions at 1733064913951 (+114 ms)Region opened successfully at 1733064913976 (+25 ms) 2024-12-01T14:55:13,976 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T14:55:13,976 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T14:55:13,977 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T14:55:13,977 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T14:55:13,977 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T14:55:13,980 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T14:55:13,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733064913976Disabling compacts and flushes for region at 1733064913976Disabling writes for close at 1733064913977 (+1 ms)Writing region close event to WAL at 1733064913979 (+2 ms)Closed at 1733064913979 2024-12-01T14:55:13,982 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T14:55:13,983 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-01T14:55:13,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T14:55:13,986 DEBUG [RS:1;519b220a2ba1:42799 {}] zookeeper.ZKUtil(111): regionserver:42799-0x10192ad7a110002, quorum=127.0.0.1:58374, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/519b220a2ba1,42799,1733064912089 2024-12-01T14:55:13,986 DEBUG [RS:0;519b220a2ba1:43229 {}] zookeeper.ZKUtil(111): regionserver:43229-0x10192ad7a110001, quorum=127.0.0.1:58374, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/519b220a2ba1,43229,1733064911911 2024-12-01T14:55:13,986 WARN [RS:0;519b220a2ba1:43229 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T14:55:13,986 WARN [RS:1;519b220a2ba1:42799 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T14:55:13,986 INFO [RS:1;519b220a2ba1:42799 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:55:13,986 INFO [RS:0;519b220a2ba1:43229 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:55:13,987 DEBUG [RS:0;519b220a2ba1:43229 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,43229,1733064911911 2024-12-01T14:55:13,987 DEBUG [RS:1;519b220a2ba1:42799 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,42799,1733064912089 2024-12-01T14:55:13,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-01T14:55:14,012 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [519b220a2ba1,43229,1733064911911] 2024-12-01T14:55:14,013 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [519b220a2ba1,42799,1733064912089] 2024-12-01T14:55:14,018 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T14:55:14,025 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-01T14:55:14,048 INFO [RS:0;519b220a2ba1:43229 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T14:55:14,049 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T14:55:14,068 INFO [RS:0;519b220a2ba1:43229 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T14:55:14,070 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T14:55:14,077 INFO [RS:0;519b220a2ba1:43229 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T14:55:14,077 INFO [RS:0;519b220a2ba1:43229 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,083 INFO [RS:1;519b220a2ba1:42799 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T14:55:14,083 INFO [RS:1;519b220a2ba1:42799 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,092 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T14:55:14,092 INFO [RS:0;519b220a2ba1:43229 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T14:55:14,103 INFO [RS:1;519b220a2ba1:42799 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T14:55:14,103 INFO [RS:0;519b220a2ba1:43229 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T14:55:14,105 INFO [RS:1;519b220a2ba1:42799 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,106 DEBUG [RS:1;519b220a2ba1:42799 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,106 DEBUG [RS:1;519b220a2ba1:42799 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,106 DEBUG [RS:1;519b220a2ba1:42799 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,106 DEBUG [RS:1;519b220a2ba1:42799 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,107 DEBUG [RS:1;519b220a2ba1:42799 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,107 DEBUG [RS:1;519b220a2ba1:42799 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/519b220a2ba1:0, corePoolSize=2, maxPoolSize=2 2024-12-01T14:55:14,107 DEBUG [RS:1;519b220a2ba1:42799 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,107 DEBUG [RS:1;519b220a2ba1:42799 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,107 DEBUG [RS:1;519b220a2ba1:42799 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,107 DEBUG [RS:1;519b220a2ba1:42799 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,108 DEBUG [RS:1;519b220a2ba1:42799 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,108 DEBUG [RS:1;519b220a2ba1:42799 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,108 DEBUG [RS:1;519b220a2ba1:42799 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/519b220a2ba1:0, corePoolSize=3, maxPoolSize=3 2024-12-01T14:55:14,108 DEBUG [RS:1;519b220a2ba1:42799 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/519b220a2ba1:0, corePoolSize=3, maxPoolSize=3 2024-12-01T14:55:14,110 INFO [RS:0;519b220a2ba1:43229 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,110 DEBUG [RS:0;519b220a2ba1:43229 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,110 DEBUG [RS:0;519b220a2ba1:43229 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,110 DEBUG [RS:0;519b220a2ba1:43229 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,111 DEBUG [RS:0;519b220a2ba1:43229 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,111 DEBUG [RS:0;519b220a2ba1:43229 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,111 DEBUG [RS:0;519b220a2ba1:43229 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/519b220a2ba1:0, corePoolSize=2, maxPoolSize=2 2024-12-01T14:55:14,111 DEBUG [RS:0;519b220a2ba1:43229 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,111 DEBUG [RS:0;519b220a2ba1:43229 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,111 DEBUG [RS:0;519b220a2ba1:43229 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,112 DEBUG [RS:0;519b220a2ba1:43229 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,112 DEBUG [RS:0;519b220a2ba1:43229 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,112 DEBUG [RS:0;519b220a2ba1:43229 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:14,112 DEBUG [RS:0;519b220a2ba1:43229 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/519b220a2ba1:0, corePoolSize=3, maxPoolSize=3 2024-12-01T14:55:14,112 DEBUG [RS:0;519b220a2ba1:43229 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/519b220a2ba1:0, corePoolSize=3, maxPoolSize=3 2024-12-01T14:55:14,124 INFO [RS:1;519b220a2ba1:42799 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,124 INFO [RS:1;519b220a2ba1:42799 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,124 INFO [RS:1;519b220a2ba1:42799 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,125 INFO [RS:1;519b220a2ba1:42799 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,125 INFO [RS:1;519b220a2ba1:42799 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,125 INFO [RS:1;519b220a2ba1:42799 {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,42799,1733064912089-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T14:55:14,143 INFO [RS:0;519b220a2ba1:43229 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,143 INFO [RS:0;519b220a2ba1:43229 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,143 INFO [RS:0;519b220a2ba1:43229 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,144 INFO [RS:0;519b220a2ba1:43229 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,144 INFO [RS:0;519b220a2ba1:43229 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,144 INFO [RS:0;519b220a2ba1:43229 {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,43229,1733064911911-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T14:55:14,158 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T14:55:14,160 INFO [RS:1;519b220a2ba1:42799 {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,42799,1733064912089-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,161 INFO [RS:1;519b220a2ba1:42799 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,161 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.Replication(171): 519b220a2ba1,42799,1733064912089 started 2024-12-01T14:55:14,169 INFO [RS:0;519b220a2ba1:43229 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T14:55:14,170 INFO [RS:0;519b220a2ba1:43229 {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,43229,1733064911911-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,172 INFO [RS:0;519b220a2ba1:43229 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,172 INFO [RS:0;519b220a2ba1:43229 {}] regionserver.Replication(171): 519b220a2ba1,43229,1733064911911 started 2024-12-01T14:55:14,177 WARN [519b220a2ba1:33719 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T14:55:14,187 INFO [RS:1;519b220a2ba1:42799 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,188 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.HRegionServer(1482): Serving as 519b220a2ba1,42799,1733064912089, RpcServer on 519b220a2ba1/172.17.0.2:42799, sessionid=0x10192ad7a110002 2024-12-01T14:55:14,189 DEBUG [RS:1;519b220a2ba1:42799 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T14:55:14,189 DEBUG [RS:1;519b220a2ba1:42799 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 519b220a2ba1,42799,1733064912089 2024-12-01T14:55:14,190 DEBUG [RS:1;519b220a2ba1:42799 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '519b220a2ba1,42799,1733064912089' 2024-12-01T14:55:14,190 DEBUG [RS:1;519b220a2ba1:42799 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-01T14:55:14,191 DEBUG [RS:1;519b220a2ba1:42799 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-01T14:55:14,192 DEBUG [RS:1;519b220a2ba1:42799 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T14:55:14,192 DEBUG [RS:1;519b220a2ba1:42799 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T14:55:14,192 DEBUG [RS:1;519b220a2ba1:42799 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 519b220a2ba1,42799,1733064912089 2024-12-01T14:55:14,192 DEBUG [RS:1;519b220a2ba1:42799 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '519b220a2ba1,42799,1733064912089' 2024-12-01T14:55:14,192 DEBUG [RS:1;519b220a2ba1:42799 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-01T14:55:14,193 DEBUG [RS:1;519b220a2ba1:42799 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-01T14:55:14,194 DEBUG [RS:1;519b220a2ba1:42799 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T14:55:14,194 INFO [RS:1;519b220a2ba1:42799 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T14:55:14,194 INFO [RS:1;519b220a2ba1:42799 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T14:55:14,202 INFO [RS:0;519b220a2ba1:43229 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,202 INFO [RS:0;519b220a2ba1:43229 {}] regionserver.HRegionServer(1482): Serving as 519b220a2ba1,43229,1733064911911, RpcServer on 519b220a2ba1/172.17.0.2:43229, sessionid=0x10192ad7a110001 2024-12-01T14:55:14,203 DEBUG [RS:0;519b220a2ba1:43229 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T14:55:14,203 DEBUG [RS:0;519b220a2ba1:43229 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 519b220a2ba1,43229,1733064911911 2024-12-01T14:55:14,203 DEBUG [RS:0;519b220a2ba1:43229 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '519b220a2ba1,43229,1733064911911' 2024-12-01T14:55:14,203 DEBUG [RS:0;519b220a2ba1:43229 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-01T14:55:14,204 DEBUG [RS:0;519b220a2ba1:43229 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-01T14:55:14,205 DEBUG [RS:0;519b220a2ba1:43229 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T14:55:14,205 DEBUG [RS:0;519b220a2ba1:43229 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T14:55:14,205 DEBUG [RS:0;519b220a2ba1:43229 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 519b220a2ba1,43229,1733064911911 2024-12-01T14:55:14,206 DEBUG [RS:0;519b220a2ba1:43229 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '519b220a2ba1,43229,1733064911911' 2024-12-01T14:55:14,206 DEBUG [RS:0;519b220a2ba1:43229 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-01T14:55:14,206 DEBUG [RS:0;519b220a2ba1:43229 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-01T14:55:14,207 DEBUG [RS:0;519b220a2ba1:43229 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T14:55:14,207 INFO [RS:0;519b220a2ba1:43229 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T14:55:14,208 INFO [RS:0;519b220a2ba1:43229 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T14:55:14,299 INFO [RS:1;519b220a2ba1:42799 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:55:14,300 INFO [RS:1;519b220a2ba1:42799 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=519b220a2ba1%2C42799%2C1733064912089, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,42799,1733064912089, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/oldWALs, maxLogs=55 2024-12-01T14:55:14,309 INFO [RS:0;519b220a2ba1:43229 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:55:14,310 INFO [RS:0;519b220a2ba1:43229 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=519b220a2ba1%2C43229%2C1733064911911, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,43229,1733064911911, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/oldWALs, maxLogs=55 2024-12-01T14:55:14,322 INFO [RS:1;519b220a2ba1:42799 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,42799,1733064912089/519b220a2ba1%2C42799%2C1733064912089.1733064914302 2024-12-01T14:55:14,323 DEBUG [RS:1;519b220a2ba1:42799 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-01T14:55:14,343 INFO [RS:0;519b220a2ba1:43229 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,43229,1733064911911/519b220a2ba1%2C43229%2C1733064911911.1733064914312 2024-12-01T14:55:14,345 DEBUG [RS:0;519b220a2ba1:43229 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-01T14:55:14,429 DEBUG [519b220a2ba1:33719 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-01T14:55:14,440 DEBUG [519b220a2ba1:33719 {}] balancer.BalancerClusterState(204): Hosts are {519b220a2ba1=0} racks are {/default-rack=0} 2024-12-01T14:55:14,443 DEBUG [519b220a2ba1:33719 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T14:55:14,443 DEBUG [519b220a2ba1:33719 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T14:55:14,443 DEBUG [519b220a2ba1:33719 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T14:55:14,443 DEBUG [519b220a2ba1:33719 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T14:55:14,443 INFO [519b220a2ba1:33719 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T14:55:14,443 INFO [519b220a2ba1:33719 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T14:55:14,444 DEBUG [519b220a2ba1:33719 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T14:55:14,451 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=519b220a2ba1,42799,1733064912089 2024-12-01T14:55:14,457 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 519b220a2ba1,42799,1733064912089, state=OPENING 2024-12-01T14:55:14,505 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-01T14:55:14,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42799-0x10192ad7a110002, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:14,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:14,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43229-0x10192ad7a110001, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:14,514 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:55:14,514 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:55:14,515 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:55:14,516 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T14:55:14,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=519b220a2ba1,42799,1733064912089}] 2024-12-01T14:55:14,689 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T14:55:14,693 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39933, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T14:55:14,702 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-01T14:55:14,703 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:55:14,704 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-01T14:55:14,705 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=519b220a2ba1%2C42799%2C1733064912089.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,42799,1733064912089, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/oldWALs, maxLogs=55 2024-12-01T14:55:14,730 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,42799,1733064912089/519b220a2ba1%2C42799%2C1733064912089.meta.1733064914708.meta 2024-12-01T14:55:14,731 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-01T14:55:14,731 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:55:14,734 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-01T14:55:14,737 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-01T14:55:14,742 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-01T14:55:14,747 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-01T14:55:14,748 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:55:14,748 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-01T14:55:14,748 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-01T14:55:14,750 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-01T14:55:14,751 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T14:55:14,751 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:14,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:14,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-01T14:55:14,752 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T14:55:14,752 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:14,753 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:14,753 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-01T14:55:14,754 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T14:55:14,754 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:14,754 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:14,754 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-01T14:55:14,755 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T14:55:14,755 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:14,756 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:14,756 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T14:55:14,757 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740 2024-12-01T14:55:14,759 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740 2024-12-01T14:55:14,759 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T14:55:14,759 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T14:55:14,759 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T14:55:14,761 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T14:55:14,762 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59104011, jitterRate=-0.11928160488605499}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T14:55:14,762 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-01T14:55:14,764 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733064914749Writing region info on filesystem at 1733064914749Initializing all the Stores at 1733064914749Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733064914749Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733064914750 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733064914750Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733064914750Cleaning up temporary data from old regions at 1733064914759 (+9 ms)Running coprocessor post-open hooks at 1733064914762 (+3 ms)Region opened successfully at 1733064914764 (+2 ms) 2024-12-01T14:55:14,771 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733064914682 2024-12-01T14:55:14,783 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-01T14:55:14,784 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-01T14:55:14,785 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=519b220a2ba1,42799,1733064912089 2024-12-01T14:55:14,787 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 519b220a2ba1,42799,1733064912089, state=OPEN 2024-12-01T14:55:14,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43229-0x10192ad7a110001, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T14:55:14,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42799-0x10192ad7a110002, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T14:55:14,814 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:55:14,814 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:55:14,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T14:55:14,814 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:55:14,814 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=519b220a2ba1,42799,1733064912089 2024-12-01T14:55:14,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-01T14:55:14,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=519b220a2ba1,42799,1733064912089 in 296 msec 2024-12-01T14:55:14,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-01T14:55:14,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 821 msec 2024-12-01T14:55:14,823 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T14:55:14,823 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-01T14:55:14,849 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T14:55:14,850 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=519b220a2ba1,42799,1733064912089, seqNum=-1] 2024-12-01T14:55:14,876 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T14:55:14,879 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46969, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T14:55:14,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2560 sec 2024-12-01T14:55:14,899 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733064914898, completionTime=-1 2024-12-01T14:55:14,902 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-01T14:55:14,902 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-01T14:55:14,934 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-01T14:55:14,934 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733064974934 2024-12-01T14:55:14,934 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733065034934 2024-12-01T14:55:14,934 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 31 msec 2024-12-01T14:55:14,937 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-01T14:55:14,943 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,33719,1733064910998-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,943 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,33719,1733064910998-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,943 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,33719,1733064910998-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,944 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-519b220a2ba1:33719, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,945 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,945 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:14,952 DEBUG [master/519b220a2ba1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-01T14:55:14,982 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.700sec 2024-12-01T14:55:14,984 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-01T14:55:14,986 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-01T14:55:14,987 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-01T14:55:14,988 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-01T14:55:14,988 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-01T14:55:14,989 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,33719,1733064910998-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T14:55:14,990 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,33719,1733064910998-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-01T14:55:14,991 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-01T14:55:14,993 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-01T14:55:14,993 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,33719,1733064910998-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:15,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73434faf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T14:55:15,104 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-01T14:55:15,104 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-01T14:55:15,113 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 519b220a2ba1,33719,-1 for getting cluster id 2024-12-01T14:55:15,116 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T14:55:15,125 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '010b4207-fc35-4102-a56d-e2ac761de6a9' 2024-12-01T14:55:15,128 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T14:55:15,128 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "010b4207-fc35-4102-a56d-e2ac761de6a9" 2024-12-01T14:55:15,128 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57909e0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T14:55:15,129 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [519b220a2ba1,33719,-1] 2024-12-01T14:55:15,132 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T14:55:15,133 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:55:15,135 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T14:55:15,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70a596f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T14:55:15,138 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T14:55:15,145 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=519b220a2ba1,42799,1733064912089, seqNum=-1] 2024-12-01T14:55:15,146 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T14:55:15,148 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T14:55:15,170 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=519b220a2ba1,33719,1733064910998 2024-12-01T14:55:15,173 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-01T14:55:15,178 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 519b220a2ba1,33719,1733064910998 2024-12-01T14:55:15,180 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2d9f3d9 2024-12-01T14:55:15,181 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-01T14:55:15,184 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-01T14:55:15,190 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33719 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733064915170', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T14:55:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33719 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733064915170 2024-12-01T14:55:15,205 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733064915170 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T14:55:15,208 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33719 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733064915170" procId is: 4 2024-12-01T14:55:15,209 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:15,211 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733064915170 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T14:55:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T14:55:15,233 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733064915170-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c2dbe80958f304655f9be9a636552ebe, NAME => 'testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733064915170', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46 2024-12-01T14:55:15,242 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733064915170-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:55:15,242 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733064915170-pool-0 {}] regionserver.HRegion(1722): Closing c2dbe80958f304655f9be9a636552ebe, disabling compactions & flushes 2024-12-01T14:55:15,242 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733064915170-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:15,242 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733064915170-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:15,242 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733064915170-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. after waiting 0 ms 2024-12-01T14:55:15,242 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733064915170-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:15,242 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733064915170-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:15,242 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733064915170-pool-0 {}] regionserver.HRegion(1676): Region close journal for c2dbe80958f304655f9be9a636552ebe: Waiting for close lock at 1733064915242Disabling compacts and flushes for region at 1733064915242Disabling writes for close at 1733064915242Writing region close event to WAL at 1733064915242Closed at 1733064915242 2024-12-01T14:55:15,244 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733064915170 execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T14:55:15,249 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733064915244"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733064915244"}]},"ts":"1733064915244"} 2024-12-01T14:55:15,254 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T14:55:15,255 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733064915170 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T14:55:15,258 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733064915170","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733064915255"}]},"ts":"1733064915255"} 2024-12-01T14:55:15,262 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733064915170, state=ENABLING in hbase:meta 2024-12-01T14:55:15,263 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {519b220a2ba1=0} racks are {/default-rack=0} 2024-12-01T14:55:15,264 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T14:55:15,264 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T14:55:15,264 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T14:55:15,264 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T14:55:15,264 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T14:55:15,265 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T14:55:15,265 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T14:55:15,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733064915170, region=c2dbe80958f304655f9be9a636552ebe, ASSIGN}] 2024-12-01T14:55:15,271 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733064915170, region=c2dbe80958f304655f9be9a636552ebe, ASSIGN 2024-12-01T14:55:15,273 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733064915170, region=c2dbe80958f304655f9be9a636552ebe, ASSIGN; state=OFFLINE, location=519b220a2ba1,43229,1733064911911; forceNewPlan=false, retain=false 2024-12-01T14:55:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T14:55:15,426 INFO [519b220a2ba1:33719 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-01T14:55:15,427 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c2dbe80958f304655f9be9a636552ebe, regionState=OPENING, regionLocation=519b220a2ba1,43229,1733064911911 2024-12-01T14:55:15,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733064915170, region=c2dbe80958f304655f9be9a636552ebe, ASSIGN because future has completed 2024-12-01T14:55:15,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2dbe80958f304655f9be9a636552ebe, server=519b220a2ba1,43229,1733064911911}] 2024-12-01T14:55:15,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T14:55:15,590 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T14:55:15,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58815, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T14:55:15,609 INFO [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:15,609 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c2dbe80958f304655f9be9a636552ebe, NAME => 'testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:55:15,610 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733064915170 c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:15,610 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:55:15,611 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:15,611 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:15,615 INFO [StoreOpener-c2dbe80958f304655f9be9a636552ebe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:15,625 INFO [StoreOpener-c2dbe80958f304655f9be9a636552ebe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2dbe80958f304655f9be9a636552ebe columnFamilyName family 2024-12-01T14:55:15,625 DEBUG [StoreOpener-c2dbe80958f304655f9be9a636552ebe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:15,631 INFO [StoreOpener-c2dbe80958f304655f9be9a636552ebe-1 {}] regionserver.HStore(327): Store=c2dbe80958f304655f9be9a636552ebe/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:55:15,634 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:15,635 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:15,635 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:15,636 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:15,636 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:15,638 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:15,651 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T14:55:15,652 INFO [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c2dbe80958f304655f9be9a636552ebe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67245454, jitterRate=0.0020353496074676514}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T14:55:15,652 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:15,653 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c2dbe80958f304655f9be9a636552ebe: Running coprocessor pre-open hook at 1733064915611Writing region info on filesystem at 1733064915611Initializing all the Stores at 1733064915612 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733064915612Cleaning up temporary data from old regions at 1733064915636 (+24 ms)Running coprocessor post-open hooks at 1733064915652 (+16 ms)Region opened successfully at 1733064915653 (+1 ms) 2024-12-01T14:55:15,655 INFO [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe., pid=6, masterSystemTime=1733064915589 2024-12-01T14:55:15,659 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:15,659 INFO [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:15,660 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c2dbe80958f304655f9be9a636552ebe, regionState=OPEN, openSeqNum=2, regionLocation=519b220a2ba1,43229,1733064911911 2024-12-01T14:55:15,664 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2dbe80958f304655f9be9a636552ebe, server=519b220a2ba1,43229,1733064911911 because future has completed 2024-12-01T14:55:15,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-01T14:55:15,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c2dbe80958f304655f9be9a636552ebe, server=519b220a2ba1,43229,1733064911911 in 233 msec 2024-12-01T14:55:15,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-01T14:55:15,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733064915170, region=c2dbe80958f304655f9be9a636552ebe, ASSIGN in 405 msec 2024-12-01T14:55:15,677 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733064915170 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T14:55:15,678 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733064915170","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733064915677"}]},"ts":"1733064915677"} 2024-12-01T14:55:15,681 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733064915170, state=ENABLED in hbase:meta 2024-12-01T14:55:15,683 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733064915170 execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T14:55:15,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733064915170 in 489 msec 2024-12-01T14:55:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T14:55:15,855 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733064915170 get assigned. Timeout = 60000ms 2024-12-01T14:55:15,855 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733064915170 completed 2024-12-01T14:55:15,858 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T14:55:15,865 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733064915170 assigned to meta. Checking AM states. 2024-12-01T14:55:15,865 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T14:55:15,866 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733064915170 assigned. 2024-12-01T14:55:15,875 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733064915170', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe., hostname=519b220a2ba1,43229,1733064911911, seqNum=2] 2024-12-01T14:55:15,876 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T14:55:15,878 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T14:55:15,889 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c2dbe80958f304655f9be9a636552ebe 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-01T14:55:15,917 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 20bbfe1945f64573b9151fd3bcec838d 2024-12-01T14:55:15,957 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe/.tmp/family/20bbfe1945f64573b9151fd3bcec838d is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-01T14:55:15,961 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe/.tmp/family/20bbfe1945f64573b9151fd3bcec838d 2024-12-01T14:55:15,998 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe/.tmp/family/20bbfe1945f64573b9151fd3bcec838d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe/family/20bbfe1945f64573b9151fd3bcec838d 2024-12-01T14:55:16,000 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe/family/20bbfe1945f64573b9151fd3bcec838d, delay=951 ms 2024-12-01T14:55:16,002 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe/family/20bbfe1945f64573b9151fd3bcec838d, entries=2, sequenceid=6, filesize=4.9 K 2024-12-01T14:55:16,008 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for c2dbe80958f304655f9be9a636552ebe in 118ms, sequenceid=6, compaction requested=false 2024-12-01T14:55:16,009 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733064915170' 2024-12-01T14:55:16,012 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c2dbe80958f304655f9be9a636552ebe: 2024-12-01T14:55:16,953 DEBUG [hfile-prefetch-1733064916000 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 20bbfe1945f64573b9151fd3bcec838d_0. 2024-12-01T14:55:16,957 DEBUG [hfile-prefetch-1733064916000 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 20bbfe1945f64573b9151fd3bcec838d_117. 2024-12-01T14:55:16,959 DEBUG [hfile-prefetch-1733064916000 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe/family/20bbfe1945f64573b9151fd3bcec838d, with total blocks 2, and data blocks 1 2024-12-01T14:55:16,959 DEBUG [hfile-prefetch-1733064916000 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-01T14:55:16,959 DEBUG [hfile-prefetch-1733064916000 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe/family/20bbfe1945f64573b9151fd3bcec838d, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-01T14:55:16,960 DEBUG [hfile-prefetch-1733064916000 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe/family/20bbfe1945f64573b9151fd3bcec838d. Will sleep 100ms and try the verification again. 2024-12-01T14:55:17,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33719 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=c2dbe80958f304655f9be9a636552ebe, source=519b220a2ba1,43229,1733064911911, destination=519b220a2ba1,42799,1733064912089, warming up region on 519b220a2ba1,42799,1733064912089 2024-12-01T14:55:17,025 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33719 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=c2dbe80958f304655f9be9a636552ebe, source=519b220a2ba1,43229,1733064911911, destination=519b220a2ba1,42799,1733064912089, running balancer 2024-12-01T14:55:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33719 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733064915170, region=c2dbe80958f304655f9be9a636552ebe, REOPEN/MOVE 2024-12-01T14:55:17,027 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733064915170, region=c2dbe80958f304655f9be9a636552ebe, REOPEN/MOVE 2024-12-01T14:55:17,027 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42799 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42799 {}] regionserver.HRegion(7855): Warmup {ENCODED => c2dbe80958f304655f9be9a636552ebe, NAME => 'testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:55:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42799 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:55:17,029 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c2dbe80958f304655f9be9a636552ebe, regionState=CLOSING, regionLocation=519b220a2ba1,43229,1733064911911 2024-12-01T14:55:17,029 INFO [StoreOpener-c2dbe80958f304655f9be9a636552ebe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:17,030 INFO [StoreOpener-c2dbe80958f304655f9be9a636552ebe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2dbe80958f304655f9be9a636552ebe columnFamilyName family 2024-12-01T14:55:17,030 DEBUG [StoreOpener-c2dbe80958f304655f9be9a636552ebe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:17,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733064915170, region=c2dbe80958f304655f9be9a636552ebe, REOPEN/MOVE because future has completed 2024-12-01T14:55:17,036 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-01T14:55:17,036 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure c2dbe80958f304655f9be9a636552ebe, server=519b220a2ba1,43229,1733064911911}] 2024-12-01T14:55:17,038 DEBUG [StoreOpener-c2dbe80958f304655f9be9a636552ebe-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe/family/20bbfe1945f64573b9151fd3bcec838d 2024-12-01T14:55:17,038 INFO [StoreOpener-c2dbe80958f304655f9be9a636552ebe-1 {}] regionserver.HStore(327): Store=c2dbe80958f304655f9be9a636552ebe/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:55:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42799 {}] regionserver.HRegion(1722): Closing c2dbe80958f304655f9be9a636552ebe, disabling compactions & flushes 2024-12-01T14:55:17,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42799 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42799 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42799 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. after waiting 0 ms 2024-12-01T14:55:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42799 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:17,039 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-01T14:55:17,040 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-01T14:55:17,040 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-01T14:55:17,040 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-01T14:55:17,044 DEBUG [StoreFileCloser-c2dbe80958f304655f9be9a636552ebe-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe/family/20bbfe1945f64573b9151fd3bcec838d 2024-12-01T14:55:17,044 WARN [hfile-prefetch-1733064916000 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe/family/20bbfe1945f64573b9151fd3bcec838d, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-01T14:55:17,044 DEBUG [hfile-prefetch-1733064916000 {}] hfile.PrefetchExecutor(121): Prefetch completed for 20bbfe1945f64573b9151fd3bcec838d 2024-12-01T14:55:17,047 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42799 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42799 {}] regionserver.HRegion(1676): Region close journal for c2dbe80958f304655f9be9a636552ebe: Waiting for close lock at 1733064917038Disabling compacts and flushes for region at 1733064917038Disabling writes for close at 1733064917039 (+1 ms)Writing region close event to WAL at 1733064917047 (+8 ms)Closed at 1733064917047 2024-12-01T14:55:17,059 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-01T14:55:17,059 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733064917040 2024-12-01T14:55:17,059 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-01T14:55:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33719 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-01T14:55:17,196 INFO [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:17,196 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-01T14:55:17,196 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing c2dbe80958f304655f9be9a636552ebe, disabling compactions & flushes 2024-12-01T14:55:17,197 INFO [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:17,197 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:17,197 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. after waiting 0 ms 2024-12-01T14:55:17,197 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:17,198 DEBUG [StoreFileCloser-c2dbe80958f304655f9be9a636552ebe-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 20bbfe1945f64573b9151fd3bcec838d, starting offset: 0, end offset: 9223372036854775807 2024-12-01T14:55:17,200 DEBUG [StoreFileCloser-c2dbe80958f304655f9be9a636552ebe-family-1 {}] bucket.BucketCache(856): removed key 20bbfe1945f64573b9151fd3bcec838d_0 from back map with offset lock 2094080 in the evict process 2024-12-01T14:55:17,205 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-01T14:55:17,206 INFO [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:17,206 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for c2dbe80958f304655f9be9a636552ebe: Waiting for close lock at 1733064917196Running coprocessor pre-close hooks at 1733064917196Disabling compacts and flushes for region at 1733064917196Disabling writes for close at 1733064917197 (+1 ms)Writing region close event to WAL at 1733064917201 (+4 ms)Running coprocessor post-close hooks at 1733064917205 (+4 ms)Closed at 1733064917206 (+1 ms) 2024-12-01T14:55:17,207 INFO [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding c2dbe80958f304655f9be9a636552ebe move to 519b220a2ba1,42799,1733064912089 record at close sequenceid=6 2024-12-01T14:55:17,210 INFO [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:17,211 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c2dbe80958f304655f9be9a636552ebe, regionState=CLOSED 2024-12-01T14:55:17,214 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure c2dbe80958f304655f9be9a636552ebe, server=519b220a2ba1,43229,1733064911911 because future has completed 2024-12-01T14:55:17,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-01T14:55:17,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure c2dbe80958f304655f9be9a636552ebe, server=519b220a2ba1,43229,1733064911911 in 179 msec 2024-12-01T14:55:17,220 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733064915170, region=c2dbe80958f304655f9be9a636552ebe, REOPEN/MOVE; state=CLOSED, location=519b220a2ba1,42799,1733064912089; forceNewPlan=false, retain=false 2024-12-01T14:55:17,221 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-01T14:55:17,221 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-01T14:55:17,222 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-01T14:55:17,222 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-01T14:55:17,226 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-01T14:55:17,226 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733064917221 2024-12-01T14:55:17,226 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-01T14:55:17,371 INFO [519b220a2ba1:33719 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-01T14:55:17,371 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c2dbe80958f304655f9be9a636552ebe, regionState=OPENING, regionLocation=519b220a2ba1,42799,1733064912089 2024-12-01T14:55:17,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733064915170, region=c2dbe80958f304655f9be9a636552ebe, REOPEN/MOVE because future has completed 2024-12-01T14:55:17,375 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2dbe80958f304655f9be9a636552ebe, server=519b220a2ba1,42799,1733064912089}] 2024-12-01T14:55:17,531 INFO [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:17,531 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => c2dbe80958f304655f9be9a636552ebe, NAME => 'testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:55:17,532 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733064915170 c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:17,532 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:55:17,532 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:17,532 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:17,533 INFO [StoreOpener-c2dbe80958f304655f9be9a636552ebe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:17,535 INFO [StoreOpener-c2dbe80958f304655f9be9a636552ebe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2dbe80958f304655f9be9a636552ebe columnFamilyName family 2024-12-01T14:55:17,535 DEBUG [StoreOpener-c2dbe80958f304655f9be9a636552ebe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:17,537 DEBUG [StoreFileOpener-c2dbe80958f304655f9be9a636552ebe-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe/family/20bbfe1945f64573b9151fd3bcec838d, delay=998 ms 2024-12-01T14:55:17,538 DEBUG [StoreOpener-c2dbe80958f304655f9be9a636552ebe-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe/family/20bbfe1945f64573b9151fd3bcec838d 2024-12-01T14:55:17,539 INFO [StoreOpener-c2dbe80958f304655f9be9a636552ebe-1 {}] regionserver.HStore(327): Store=c2dbe80958f304655f9be9a636552ebe/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:55:17,539 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:17,540 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:17,541 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:17,541 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:17,541 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:17,545 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:17,546 INFO [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened c2dbe80958f304655f9be9a636552ebe; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74627158, jitterRate=0.11203131079673767}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T14:55:17,546 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:17,548 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for c2dbe80958f304655f9be9a636552ebe: Running coprocessor pre-open hook at 1733064917532Writing region info on filesystem at 1733064917533 (+1 ms)Initializing all the Stores at 1733064917533Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733064917533Cleaning up temporary data from old regions at 1733064917541 (+8 ms)Running coprocessor post-open hooks at 1733064917547 (+6 ms)Region opened successfully at 1733064917548 (+1 ms) 2024-12-01T14:55:17,549 INFO [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe., pid=9, masterSystemTime=1733064917527 2024-12-01T14:55:17,552 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:17,552 INFO [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:17,552 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c2dbe80958f304655f9be9a636552ebe, regionState=OPEN, openSeqNum=10, regionLocation=519b220a2ba1,42799,1733064912089 2024-12-01T14:55:17,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2dbe80958f304655f9be9a636552ebe, server=519b220a2ba1,42799,1733064912089 because future has completed 2024-12-01T14:55:17,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-01T14:55:17,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure c2dbe80958f304655f9be9a636552ebe, server=519b220a2ba1,42799,1733064912089 in 181 msec 2024-12-01T14:55:17,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733064915170, region=c2dbe80958f304655f9be9a636552ebe, REOPEN/MOVE in 533 msec 2024-12-01T14:55:17,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-01T14:55:17,580 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T14:55:17,580 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T14:55:17,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:55:17,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:55:17,586 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T14:55:17,586 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-01T14:55:17,586 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1058988911, stopped=false 2024-12-01T14:55:17,587 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=519b220a2ba1,33719,1733064910998 2024-12-01T14:55:17,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42799-0x10192ad7a110002, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T14:55:17,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43229-0x10192ad7a110001, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T14:55:17,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42799-0x10192ad7a110002, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:17,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43229-0x10192ad7a110001, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:17,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T14:55:17,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:17,613 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-01T14:55:17,614 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42799-0x10192ad7a110002, quorum=127.0.0.1:58374, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T14:55:17,614 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43229-0x10192ad7a110001, quorum=127.0.0.1:58374, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T14:55:17,614 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T14:55:17,614 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T14:55:17,614 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T14:55:17,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:55:17,615 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '519b220a2ba1,43229,1733064911911' ***** 2024-12-01T14:55:17,615 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-01T14:55:17,615 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '519b220a2ba1,42799,1733064912089' ***** 2024-12-01T14:55:17,615 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-01T14:55:17,615 INFO [RS:1;519b220a2ba1:42799 {}] bucket.BucketCache(1738): Disabling cache 2024-12-01T14:55:17,615 INFO [RS:0;519b220a2ba1:43229 {}] bucket.BucketCache(1738): Disabling cache 2024-12-01T14:55:17,616 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-01T14:55:17,616 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-01T14:55:17,616 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-01T14:55:17,616 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-01T14:55:17,616 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-01T14:55:17,616 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-01T14:55:17,616 INFO [RS:0;519b220a2ba1:43229 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/bucket.persistence 2024-12-01T14:55:17,616 INFO [RS:1;519b220a2ba1:42799 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/bucket.persistence 2024-12-01T14:55:17,616 INFO [RS:0;519b220a2ba1:43229 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-01T14:55:17,616 INFO [RS:1;519b220a2ba1:42799 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-01T14:55:17,616 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-01T14:55:17,616 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-01T14:55:17,626 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T14:55:17,627 DEBUG [RS:0;519b220a2ba1:43229 {}] bucket.BucketCache(1487): Thread RS:0;519b220a2ba1:43229 started persisting bucket cache to file 2024-12-01T14:55:17,627 DEBUG [RS:1;519b220a2ba1:42799 {}] bucket.BucketCache(1487): Thread RS:1;519b220a2ba1:42799 started persisting bucket cache to file 2024-12-01T14:55:17,627 DEBUG [RS:0;519b220a2ba1:43229 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-01T14:55:17,627 DEBUG [RS:0;519b220a2ba1:43229 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-01T14:55:17,627 DEBUG [RS:1;519b220a2ba1:42799 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-01T14:55:17,627 DEBUG [RS:1;519b220a2ba1:42799 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-01T14:55:17,631 DEBUG [RS:1;519b220a2ba1:42799 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-01T14:55:17,631 DEBUG [RS:1;519b220a2ba1:42799 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733064917627 2024-12-01T14:55:17,633 WARN [Thread-95 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-12-01T14:55:17,633 DEBUG [RS:0;519b220a2ba1:43229 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-01T14:55:17,633 DEBUG [RS:0;519b220a2ba1:43229 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733064917627 2024-12-01T14:55:17,633 DEBUG [RS:0;519b220a2ba1:43229 {}] bucket.BucketCache(1509): Thread RS:0;519b220a2ba1:43229 finished persisting bucket cache to file, renaming 2024-12-01T14:55:17,634 INFO [RS:0;519b220a2ba1:43229 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T14:55:17,634 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T14:55:17,634 INFO [RS:0;519b220a2ba1:43229 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-01T14:55:17,634 INFO [RS:0;519b220a2ba1:43229 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-01T14:55:17,634 INFO [RS:0;519b220a2ba1:43229 {}] regionserver.HRegionServer(959): stopping server 519b220a2ba1,43229,1733064911911 2024-12-01T14:55:17,634 INFO [RS:0;519b220a2ba1:43229 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T14:55:17,634 INFO [regionserver/519b220a2ba1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T14:55:17,634 INFO [RS:0;519b220a2ba1:43229 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;519b220a2ba1:43229. 2024-12-01T14:55:17,635 DEBUG [RS:0;519b220a2ba1:43229 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T14:55:17,635 DEBUG [RS:0;519b220a2ba1:43229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:55:17,635 INFO [RS:0;519b220a2ba1:43229 {}] regionserver.HRegionServer(976): stopping server 519b220a2ba1,43229,1733064911911; all regions closed. 2024-12-01T14:55:17,639 DEBUG [RS:0;519b220a2ba1:43229 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/oldWALs 2024-12-01T14:55:17,640 INFO [RS:0;519b220a2ba1:43229 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 519b220a2ba1%2C43229%2C1733064911911:(num 1733064914312) 2024-12-01T14:55:17,640 DEBUG [RS:0;519b220a2ba1:43229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:55:17,640 INFO [RS:0;519b220a2ba1:43229 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T14:55:17,640 INFO [RS:0;519b220a2ba1:43229 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T14:55:17,640 INFO [RS:0;519b220a2ba1:43229 {}] hbase.ChoreService(370): Chore service for: regionserver/519b220a2ba1:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-01T14:55:17,640 INFO [RS:0;519b220a2ba1:43229 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T14:55:17,641 INFO [RS:0;519b220a2ba1:43229 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T14:55:17,641 INFO [regionserver/519b220a2ba1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T14:55:17,641 INFO [RS:0;519b220a2ba1:43229 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T14:55:17,641 INFO [RS:0;519b220a2ba1:43229 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T14:55:17,641 INFO [RS:0;519b220a2ba1:43229 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43229 2024-12-01T14:55:17,649 DEBUG [RS:1;519b220a2ba1:42799 {}] bucket.BucketCache(1509): Thread RS:1;519b220a2ba1:42799 finished persisting bucket cache to file, renaming 2024-12-01T14:55:17,649 WARN [RS:1;519b220a2ba1:42799 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-01T14:55:17,649 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T14:55:17,649 INFO [RS:1;519b220a2ba1:42799 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-01T14:55:17,649 INFO [RS:1;519b220a2ba1:42799 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-01T14:55:17,649 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.HRegionServer(3091): Received CLOSE for c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:17,650 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.HRegionServer(959): stopping server 519b220a2ba1,42799,1733064912089 2024-12-01T14:55:17,650 INFO [RS:1;519b220a2ba1:42799 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T14:55:17,650 INFO [RS:1;519b220a2ba1:42799 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;519b220a2ba1:42799. 2024-12-01T14:55:17,650 DEBUG [RS:1;519b220a2ba1:42799 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T14:55:17,650 DEBUG [RS:1;519b220a2ba1:42799 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:55:17,650 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T14:55:17,650 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T14:55:17,651 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T14:55:17,650 INFO [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-01T14:55:17,651 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-01T14:55:17,651 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c2dbe80958f304655f9be9a636552ebe, disabling compactions & flushes 2024-12-01T14:55:17,651 INFO [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:17,651 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:17,651 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-01T14:55:17,652 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. after waiting 0 ms 2024-12-01T14:55:17,652 DEBUG [RS:1;519b220a2ba1:42799 {}] regionserver.HRegionServer(1325): Online Regions={c2dbe80958f304655f9be9a636552ebe=testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe., 1588230740=hbase:meta,,1.1588230740} 2024-12-01T14:55:17,652 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:17,652 DEBUG [RS:1;519b220a2ba1:42799 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, c2dbe80958f304655f9be9a636552ebe 2024-12-01T14:55:17,652 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-01T14:55:17,652 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T14:55:17,652 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T14:55:17,652 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T14:55:17,653 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T14:55:17,653 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T14:55:17,653 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-01T14:55:17,659 DEBUG [StoreFileCloser-c2dbe80958f304655f9be9a636552ebe-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe/family/20bbfe1945f64573b9151fd3bcec838d 2024-12-01T14:55:17,662 INFO [regionserver/519b220a2ba1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T14:55:17,668 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnRegionMove1733064915170/c2dbe80958f304655f9be9a636552ebe/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-01T14:55:17,668 INFO [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:17,668 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c2dbe80958f304655f9be9a636552ebe: Waiting for close lock at 1733064917650Running coprocessor pre-close hooks at 1733064917651 (+1 ms)Disabling compacts and flushes for region at 1733064917651Disabling writes for close at 1733064917652 (+1 ms)Writing region close event to WAL at 1733064917659 (+7 ms)Running coprocessor post-close hooks at 1733064917668 (+9 ms)Closed at 1733064917668 2024-12-01T14:55:17,669 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe. 2024-12-01T14:55:17,671 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8b830541daa34671b319e640e25086d8 2024-12-01T14:55:17,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43229-0x10192ad7a110001, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/519b220a2ba1,43229,1733064911911 2024-12-01T14:55:17,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T14:55:17,685 INFO [RS:0;519b220a2ba1:43229 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T14:55:17,686 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [519b220a2ba1,43229,1733064911911] 2024-12-01T14:55:17,688 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/info/8b830541daa34671b319e640e25086d8 is 205, key is testBlockEvictionOnRegionMove1733064915170,,1733064915186.c2dbe80958f304655f9be9a636552ebe./info:regioninfo/1733064917552/Put/seqid=0 2024-12-01T14:55:17,689 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/info/8b830541daa34671b319e640e25086d8 2024-12-01T14:55:17,701 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 554175f11055487c844b095abf282560 2024-12-01T14:55:17,704 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/519b220a2ba1,43229,1733064911911 already deleted, retry=false 2024-12-01T14:55:17,704 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 519b220a2ba1,43229,1733064911911 expired; onlineServers=1 2024-12-01T14:55:17,720 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/ns/554175f11055487c844b095abf282560 is 43, key is default/ns:d/1733064914884/Put/seqid=0 2024-12-01T14:55:17,721 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/ns/554175f11055487c844b095abf282560 2024-12-01T14:55:17,732 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 31875cd2d5a04b72bbea6cb1d507d628 2024-12-01T14:55:17,750 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/table/31875cd2d5a04b72bbea6cb1d507d628 is 78, key is testBlockEvictionOnRegionMove1733064915170/table:state/1733064915677/Put/seqid=0 2024-12-01T14:55:17,751 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/table/31875cd2d5a04b72bbea6cb1d507d628 2024-12-01T14:55:17,755 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/info/8b830541daa34671b319e640e25086d8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/info/8b830541daa34671b319e640e25086d8 2024-12-01T14:55:17,756 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/info/8b830541daa34671b319e640e25086d8, delay=967 ms 2024-12-01T14:55:17,758 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/info/8b830541daa34671b319e640e25086d8, entries=15, sequenceid=15, filesize=7.6 K 2024-12-01T14:55:17,759 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/ns/554175f11055487c844b095abf282560 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/ns/554175f11055487c844b095abf282560 2024-12-01T14:55:17,760 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/ns/554175f11055487c844b095abf282560, delay=902 ms 2024-12-01T14:55:17,762 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/ns/554175f11055487c844b095abf282560, entries=2, sequenceid=15, filesize=5.0 K 2024-12-01T14:55:17,763 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/table/31875cd2d5a04b72bbea6cb1d507d628 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/table/31875cd2d5a04b72bbea6cb1d507d628 2024-12-01T14:55:17,764 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/table/31875cd2d5a04b72bbea6cb1d507d628, delay=980 ms 2024-12-01T14:55:17,766 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/table/31875cd2d5a04b72bbea6cb1d507d628, entries=2, sequenceid=15, filesize=5.3 K 2024-12-01T14:55:17,768 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 115ms, sequenceid=15, compaction requested=false 2024-12-01T14:55:17,768 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-01T14:55:17,770 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/info/8b830541daa34671b319e640e25086d8 2024-12-01T14:55:17,771 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/ns/554175f11055487c844b095abf282560 2024-12-01T14:55:17,772 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/table/31875cd2d5a04b72bbea6cb1d507d628 2024-12-01T14:55:17,777 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-01T14:55:17,778 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T14:55:17,778 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T14:55:17,778 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733064917652Running coprocessor pre-close hooks at 1733064917652Disabling compacts and flushes for region at 1733064917652Disabling writes for close at 1733064917653 (+1 ms)Obtaining lock to block concurrent updates at 1733064917653Preparing flush snapshotting stores in 1588230740 at 1733064917653Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733064917654 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733064917654Flushing 1588230740/info: creating writer at 1733064917655 (+1 ms)Flushing 1588230740/info: appending metadata at 1733064917687 (+32 ms)Flushing 1588230740/info: closing flushed file at 1733064917687Flushing 1588230740/ns: creating writer at 1733064917693 (+6 ms)Flushing 1588230740/ns: appending metadata at 1733064917719 (+26 ms)Flushing 1588230740/ns: closing flushed file at 1733064917719Flushing 1588230740/table: creating writer at 1733064917724 (+5 ms)Flushing 1588230740/table: appending metadata at 1733064917749 (+25 ms)Flushing 1588230740/table: closing flushed file at 1733064917749Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29ba1263: reopening flushed file at 1733064917754 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f210775: reopening flushed file at 1733064917758 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9558811: reopening flushed file at 1733064917762 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 115ms, sequenceid=15, compaction requested=false at 1733064917768 (+6 ms)Writing region close event to WAL at 1733064917772 (+4 ms)Running coprocessor post-close hooks at 1733064917777 (+5 ms)Closed at 1733064917778 (+1 ms) 2024-12-01T14:55:17,779 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-01T14:55:17,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43229-0x10192ad7a110001, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:55:17,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43229-0x10192ad7a110001, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:55:17,798 INFO [RS:0;519b220a2ba1:43229 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T14:55:17,798 INFO [RS:0;519b220a2ba1:43229 {}] regionserver.HRegionServer(1031): Exiting; stopping=519b220a2ba1,43229,1733064911911; zookeeper connection closed. 2024-12-01T14:55:17,798 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@174533c5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@174533c5 2024-12-01T14:55:17,852 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.HRegionServer(976): stopping server 519b220a2ba1,42799,1733064912089; all regions closed. 2024-12-01T14:55:17,855 DEBUG [RS:1;519b220a2ba1:42799 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/oldWALs 2024-12-01T14:55:17,855 INFO [RS:1;519b220a2ba1:42799 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 519b220a2ba1%2C42799%2C1733064912089.meta:.meta(num 1733064914708) 2024-12-01T14:55:17,857 DEBUG [RS:1;519b220a2ba1:42799 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/oldWALs 2024-12-01T14:55:17,857 INFO [RS:1;519b220a2ba1:42799 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 519b220a2ba1%2C42799%2C1733064912089:(num 1733064914302) 2024-12-01T14:55:17,857 DEBUG [RS:1;519b220a2ba1:42799 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:55:17,857 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T14:55:17,857 INFO [RS:1;519b220a2ba1:42799 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T14:55:17,857 INFO [RS:1;519b220a2ba1:42799 {}] hbase.ChoreService(370): Chore service for: regionserver/519b220a2ba1:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-01T14:55:17,858 INFO [RS:1;519b220a2ba1:42799 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T14:55:17,858 INFO [regionserver/519b220a2ba1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T14:55:17,858 INFO [RS:1;519b220a2ba1:42799 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42799 2024-12-01T14:55:17,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T14:55:17,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42799-0x10192ad7a110002, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/519b220a2ba1,42799,1733064912089 2024-12-01T14:55:17,890 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [519b220a2ba1,42799,1733064912089] 2024-12-01T14:55:17,891 INFO [RS:1;519b220a2ba1:42799 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T14:55:17,901 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/519b220a2ba1,42799,1733064912089 already deleted, retry=false 2024-12-01T14:55:17,901 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 519b220a2ba1,42799,1733064912089 expired; onlineServers=0 2024-12-01T14:55:17,902 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '519b220a2ba1,33719,1733064910998' ***** 2024-12-01T14:55:17,902 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-01T14:55:17,902 INFO [M:0;519b220a2ba1:33719 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T14:55:17,902 INFO [M:0;519b220a2ba1:33719 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T14:55:17,902 DEBUG [M:0;519b220a2ba1:33719 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-01T14:55:17,903 DEBUG [M:0;519b220a2ba1:33719 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-01T14:55:17,903 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-01T14:55:17,903 DEBUG [master/519b220a2ba1:0:becomeActiveMaster-HFileCleaner.small.0-1733064913799 {}] cleaner.HFileCleaner(306): Exit Thread[master/519b220a2ba1:0:becomeActiveMaster-HFileCleaner.small.0-1733064913799,5,FailOnTimeoutGroup] 2024-12-01T14:55:17,903 DEBUG [master/519b220a2ba1:0:becomeActiveMaster-HFileCleaner.large.0-1733064913783 {}] cleaner.HFileCleaner(306): Exit Thread[master/519b220a2ba1:0:becomeActiveMaster-HFileCleaner.large.0-1733064913783,5,FailOnTimeoutGroup] 2024-12-01T14:55:17,903 INFO [M:0;519b220a2ba1:33719 {}] hbase.ChoreService(370): Chore service for: master/519b220a2ba1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-01T14:55:17,903 INFO [M:0;519b220a2ba1:33719 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T14:55:17,903 DEBUG [M:0;519b220a2ba1:33719 {}] master.HMaster(1795): Stopping service threads 2024-12-01T14:55:17,903 INFO [M:0;519b220a2ba1:33719 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-01T14:55:17,903 INFO [M:0;519b220a2ba1:33719 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-01T14:55:17,904 INFO [M:0;519b220a2ba1:33719 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-01T14:55:17,904 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-01T14:55:17,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-01T14:55:17,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:17,921 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-12-01T14:55:17,922 DEBUG [RegionServerTracker-0 {}] master.ActiveMasterManager(353): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-12-01T14:55:17,922 INFO [M:0;519b220a2ba1:33719 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/.lastflushedseqids 2024-12-01T14:55:17,949 INFO [M:0;519b220a2ba1:33719 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-01T14:55:17,949 INFO [M:0;519b220a2ba1:33719 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-01T14:55:17,950 DEBUG [M:0;519b220a2ba1:33719 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-01T14:55:17,950 INFO [M:0;519b220a2ba1:33719 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:55:17,950 DEBUG [M:0;519b220a2ba1:33719 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:55:17,950 DEBUG [M:0;519b220a2ba1:33719 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-01T14:55:17,950 DEBUG [M:0;519b220a2ba1:33719 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:55:17,953 INFO [M:0;519b220a2ba1:33719 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.66 KB heapSize=42.99 KB 2024-12-01T14:55:17,969 DEBUG [M:0;519b220a2ba1:33719 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 102a14b6e0514278b454a93bf2afea9d 2024-12-01T14:55:17,986 DEBUG [M:0;519b220a2ba1:33719 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/102a14b6e0514278b454a93bf2afea9d is 82, key is hbase:meta,,1/info:regioninfo/1733064914784/Put/seqid=0 2024-12-01T14:55:17,987 INFO [M:0;519b220a2ba1:33719 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/102a14b6e0514278b454a93bf2afea9d 2024-12-01T14:55:17,996 DEBUG [M:0;519b220a2ba1:33719 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1e79998dbedf4173969ecd4b8a51a51c 2024-12-01T14:55:18,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42799-0x10192ad7a110002, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:55:18,010 INFO [RS:1;519b220a2ba1:42799 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T14:55:18,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42799-0x10192ad7a110002, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:55:18,010 INFO [RS:1;519b220a2ba1:42799 {}] regionserver.HRegionServer(1031): Exiting; stopping=519b220a2ba1,42799,1733064912089; zookeeper connection closed. 2024-12-01T14:55:18,010 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@707b4379 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@707b4379 2024-12-01T14:55:18,011 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-01T14:55:18,015 DEBUG [M:0;519b220a2ba1:33719 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e79998dbedf4173969ecd4b8a51a51c is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733064915685/Put/seqid=0 2024-12-01T14:55:18,016 INFO [M:0;519b220a2ba1:33719 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.04 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e79998dbedf4173969ecd4b8a51a51c 2024-12-01T14:55:18,024 DEBUG [M:0;519b220a2ba1:33719 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 60ade84ba7ef476b827b5127dcba5409 2024-12-01T14:55:18,041 DEBUG [M:0;519b220a2ba1:33719 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/60ade84ba7ef476b827b5127dcba5409 is 69, key is 519b220a2ba1,42799,1733064912089/rs:state/1733064913919/Put/seqid=0 2024-12-01T14:55:18,042 INFO [M:0;519b220a2ba1:33719 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/60ade84ba7ef476b827b5127dcba5409 2024-12-01T14:55:18,045 DEBUG [M:0;519b220a2ba1:33719 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/102a14b6e0514278b454a93bf2afea9d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/102a14b6e0514278b454a93bf2afea9d 2024-12-01T14:55:18,047 INFO [M:0;519b220a2ba1:33719 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/102a14b6e0514278b454a93bf2afea9d, entries=8, sequenceid=84, filesize=5.5 K 2024-12-01T14:55:18,048 DEBUG [M:0;519b220a2ba1:33719 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e79998dbedf4173969ecd4b8a51a51c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1e79998dbedf4173969ecd4b8a51a51c 2024-12-01T14:55:18,050 INFO [M:0;519b220a2ba1:33719 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1e79998dbedf4173969ecd4b8a51a51c, entries=9, sequenceid=84, filesize=6.5 K 2024-12-01T14:55:18,051 DEBUG [M:0;519b220a2ba1:33719 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/60ade84ba7ef476b827b5127dcba5409 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/60ade84ba7ef476b827b5127dcba5409 2024-12-01T14:55:18,053 INFO [M:0;519b220a2ba1:33719 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/60ade84ba7ef476b827b5127dcba5409, entries=2, sequenceid=84, filesize=5.1 K 2024-12-01T14:55:18,054 INFO [M:0;519b220a2ba1:33719 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=84, compaction requested=false 2024-12-01T14:55:18,057 INFO [M:0;519b220a2ba1:33719 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:55:18,058 DEBUG [M:0;519b220a2ba1:33719 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733064917949Disabling compacts and flushes for region at 1733064917949Disabling writes for close at 1733064917950 (+1 ms)Obtaining lock to block concurrent updates at 1733064917953 (+3 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733064917953Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35494, getHeapSize=43960, getOffHeapSize=0, getCellsCount=99 at 1733064917953Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733064917954 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733064917954Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733064917986 (+32 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733064917986Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733064917990 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733064918014 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733064918014Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733064918017 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733064918041 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733064918041Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@699f71b4: reopening flushed file at 1733064918045 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@173598f9: reopening flushed file at 1733064918047 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1baad2f5: reopening flushed file at 1733064918050 (+3 ms)Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=84, compaction requested=false at 1733064918054 (+4 ms)Writing region close event to WAL at 1733064918057 (+3 ms)Closed at 1733064918057 2024-12-01T14:55:18,061 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T14:55:18,061 INFO [M:0;519b220a2ba1:33719 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-01T14:55:18,061 INFO [M:0;519b220a2ba1:33719 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33719 2024-12-01T14:55:18,062 INFO [M:0;519b220a2ba1:33719 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T14:55:18,171 INFO [M:0;519b220a2ba1:33719 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T14:55:18,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:55:18,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33719-0x10192ad7a110000, quorum=127.0.0.1:58374, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:55:18,176 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-01T14:55:18,197 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-01T14:55:18,198 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:18,206 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/519b220a2ba1:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/519b220a2ba1:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: master/519b220a2ba1:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733064916000 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733064917760 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: regionserver/519b220a2ba1:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733064917757 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733064917537 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=447 (was 381) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5150 (was 5520) 2024-12-01T14:55:18,212 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=447, ProcessCount=11, AvailableMemoryMB=5150 2024-12-01T14:55:18,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/hadoop-log-dir so I do NOT create it in target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1 2024-12-01T14:55:18,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/hadoop-tmp-dir so I do NOT create it in target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1 2024-12-01T14:55:18,212 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:18,215 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1/cluster_07165460-eae2-dbf8-71a0-e04fecda8855, deleteOnExit=true 2024-12-01T14:55:18,222 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1/cluster_07165460-eae2-dbf8-71a0-e04fecda8855/zookeeper_0, clientPort=58811, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1/cluster_07165460-eae2-dbf8-71a0-e04fecda8855/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1/cluster_07165460-eae2-dbf8-71a0-e04fecda8855/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-01T14:55:18,226 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58811 2024-12-01T14:55:18,244 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46 with version=8 2024-12-01T14:55:18,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/hbase-staging 2024-12-01T14:55:18,247 INFO [Time-limited test {}] client.ConnectionUtils(128): master/519b220a2ba1:0 server-side Connection retries=45 2024-12-01T14:55:18,247 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:18,248 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:18,248 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T14:55:18,249 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:18,249 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T14:55:18,249 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-01T14:55:18,249 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T14:55:18,250 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42737 2024-12-01T14:55:18,251 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42737 connecting to ZooKeeper ensemble=127.0.0.1:58811 2024-12-01T14:55:18,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:427370x0, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T14:55:18,306 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42737-0x10192ad978f0000 connected 2024-12-01T14:55:18,380 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:18,381 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:18,383 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T14:55:18,383 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46, hbase.cluster.distributed=false 2024-12-01T14:55:18,385 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T14:55:18,387 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42737 2024-12-01T14:55:18,391 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42737 2024-12-01T14:55:18,391 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42737 2024-12-01T14:55:18,392 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42737 2024-12-01T14:55:18,392 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42737 2024-12-01T14:55:18,408 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/519b220a2ba1:0 server-side Connection retries=45 2024-12-01T14:55:18,408 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:18,408 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:18,408 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T14:55:18,408 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:18,408 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T14:55:18,408 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T14:55:18,408 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T14:55:18,409 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34279 2024-12-01T14:55:18,410 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34279 connecting to ZooKeeper ensemble=127.0.0.1:58811 2024-12-01T14:55:18,410 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:18,411 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:18,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:342790x0, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T14:55:18,422 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34279-0x10192ad978f0001 connected 2024-12-01T14:55:18,422 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T14:55:18,422 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T14:55:18,423 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1/bucket.cache 2024-12-01T14:55:18,423 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-01T14:55:18,423 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-01T14:55:18,423 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-01T14:55:18,423 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-01T14:55:18,423 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-01T14:55:18,423 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-01T14:55:18,424 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T14:55:18,424 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-01T14:55:18,424 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1/bucket.persistence is complete. 2024-12-01T14:55:18,425 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-01T14:55:18,427 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T14:55:18,429 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34279 2024-12-01T14:55:18,429 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34279 2024-12-01T14:55:18,429 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34279 2024-12-01T14:55:18,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34279 2024-12-01T14:55:18,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34279 2024-12-01T14:55:18,445 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/519b220a2ba1:0 server-side Connection retries=45 2024-12-01T14:55:18,445 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:18,445 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:18,445 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T14:55:18,445 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:18,445 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T14:55:18,445 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T14:55:18,445 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T14:55:18,446 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40943 2024-12-01T14:55:18,447 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40943 connecting to ZooKeeper ensemble=127.0.0.1:58811 2024-12-01T14:55:18,447 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:18,447 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:18,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:409430x0, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T14:55:18,455 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40943-0x10192ad978f0002 connected 2024-12-01T14:55:18,455 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40943-0x10192ad978f0002, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T14:55:18,456 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T14:55:18,457 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1/bucket.cache 2024-12-01T14:55:18,457 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-01T14:55:18,457 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-01T14:55:18,457 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-01T14:55:18,457 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-01T14:55:18,457 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-01T14:55:18,458 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-01T14:55:18,458 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1/bucket.persistence is complete. 2024-12-01T14:55:18,458 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-01T14:55:18,458 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T14:55:18,459 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40943-0x10192ad978f0002, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-01T14:55:18,460 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40943-0x10192ad978f0002, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T14:55:18,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40943 2024-12-01T14:55:18,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40943 2024-12-01T14:55:18,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40943 2024-12-01T14:55:18,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40943 2024-12-01T14:55:18,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40943 2024-12-01T14:55:18,481 DEBUG [M:0;519b220a2ba1:42737 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;519b220a2ba1:42737 2024-12-01T14:55:18,481 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/519b220a2ba1,42737,1733064918246 2024-12-01T14:55:18,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T14:55:18,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T14:55:18,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40943-0x10192ad978f0002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T14:55:18,489 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/519b220a2ba1,42737,1733064918246 2024-12-01T14:55:18,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-01T14:55:18,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:18,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40943-0x10192ad978f0002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-01T14:55:18,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:18,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40943-0x10192ad978f0002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:18,497 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-01T14:55:18,498 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/519b220a2ba1,42737,1733064918246 from backup master directory 2024-12-01T14:55:18,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40943-0x10192ad978f0002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T14:55:18,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/519b220a2ba1,42737,1733064918246 2024-12-01T14:55:18,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T14:55:18,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T14:55:18,505 WARN [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T14:55:18,505 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=519b220a2ba1,42737,1733064918246 2024-12-01T14:55:18,506 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/hbase.id] with ID: bcc9be81-f2fd-46ef-ab98-7c66ad66ea6c 2024-12-01T14:55:18,506 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/.tmp/hbase.id 2024-12-01T14:55:18,509 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/hbase.id] 2024-12-01T14:55:18,539 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:18,540 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-01T14:55:18,542 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-12-01T14:55:18,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40943-0x10192ad978f0002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:18,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:18,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:18,560 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T14:55:18,561 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-01T14:55:18,561 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:55:18,583 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store 2024-12-01T14:55:18,591 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:55:18,591 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-01T14:55:18,592 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:55:18,592 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:55:18,592 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-01T14:55:18,592 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:55:18,592 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:55:18,592 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733064918591Disabling compacts and flushes for region at 1733064918591Disabling writes for close at 1733064918592 (+1 ms)Writing region close event to WAL at 1733064918592Closed at 1733064918592 2024-12-01T14:55:18,592 WARN [master/519b220a2ba1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/.initializing 2024-12-01T14:55:18,593 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/WALs/519b220a2ba1,42737,1733064918246 2024-12-01T14:55:18,599 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:55:18,604 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=519b220a2ba1%2C42737%2C1733064918246, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/WALs/519b220a2ba1,42737,1733064918246, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/oldWALs, maxLogs=10 2024-12-01T14:55:18,622 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/WALs/519b220a2ba1,42737,1733064918246/519b220a2ba1%2C42737%2C1733064918246.1733064918604 2024-12-01T14:55:18,622 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-01T14:55:18,622 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:55:18,623 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:55:18,623 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:18,623 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:18,624 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:18,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-01T14:55:18,632 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:18,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:18,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:18,637 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-01T14:55:18,637 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:18,637 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:55:18,637 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:18,641 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-01T14:55:18,641 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:18,641 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:55:18,641 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:18,644 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-01T14:55:18,644 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:18,645 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:55:18,645 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:18,646 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:18,646 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:18,646 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:18,646 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:18,646 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T14:55:18,647 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:55:18,653 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T14:55:18,653 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61545226, jitterRate=-0.08290466666221619}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T14:55:18,654 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733064918623Initializing all the Stores at 1733064918623Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733064918623Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733064918623Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733064918623Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733064918623Cleaning up temporary data from old regions at 1733064918646 (+23 ms)Region opened successfully at 1733064918654 (+8 ms) 2024-12-01T14:55:18,655 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-01T14:55:18,659 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@346c6e54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=519b220a2ba1/172.17.0.2:0 2024-12-01T14:55:18,661 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-01T14:55:18,661 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-01T14:55:18,661 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-01T14:55:18,661 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-01T14:55:18,661 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-01T14:55:18,662 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-01T14:55:18,662 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-01T14:55:18,664 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-01T14:55:18,665 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-01T14:55:18,688 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-01T14:55:18,689 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-01T14:55:18,690 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-01T14:55:18,696 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-01T14:55:18,697 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-01T14:55:18,698 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-01T14:55:18,705 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-01T14:55:18,706 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-01T14:55:18,713 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-01T14:55:18,718 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-01T14:55:18,726 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-01T14:55:18,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40943-0x10192ad978f0002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T14:55:18,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T14:55:18,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T14:55:18,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:18,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40943-0x10192ad978f0002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:18,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:18,740 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=519b220a2ba1,42737,1733064918246, sessionid=0x10192ad978f0000, setting cluster-up flag (Was=false) 2024-12-01T14:55:18,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:18,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:18,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40943-0x10192ad978f0002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:18,788 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-01T14:55:18,790 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=519b220a2ba1,42737,1733064918246 2024-12-01T14:55:18,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:18,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:18,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40943-0x10192ad978f0002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:18,838 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-01T14:55:18,840 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=519b220a2ba1,42737,1733064918246 2024-12-01T14:55:18,840 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-01T14:55:18,841 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-01T14:55:18,842 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-01T14:55:18,842 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-01T14:55:18,842 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 519b220a2ba1,42737,1733064918246 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-01T14:55:18,844 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/519b220a2ba1:0, corePoolSize=5, maxPoolSize=5 2024-12-01T14:55:18,844 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/519b220a2ba1:0, corePoolSize=5, maxPoolSize=5 2024-12-01T14:55:18,844 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/519b220a2ba1:0, corePoolSize=5, maxPoolSize=5 2024-12-01T14:55:18,844 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/519b220a2ba1:0, corePoolSize=5, maxPoolSize=5 2024-12-01T14:55:18,844 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/519b220a2ba1:0, corePoolSize=10, maxPoolSize=10 2024-12-01T14:55:18,844 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,844 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/519b220a2ba1:0, corePoolSize=2, maxPoolSize=2 2024-12-01T14:55:18,844 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,846 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733064948846 2024-12-01T14:55:18,846 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T14:55:18,846 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-01T14:55:18,846 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-01T14:55:18,846 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-01T14:55:18,846 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-01T14:55:18,846 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-01T14:55:18,846 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-01T14:55:18,846 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-01T14:55:18,846 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:18,846 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,846 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-01T14:55:18,847 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-01T14:55:18,847 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-01T14:55:18,847 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-01T14:55:18,847 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-01T14:55:18,847 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-01T14:55:18,848 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/519b220a2ba1:0:becomeActiveMaster-HFileCleaner.large.0-1733064918847,5,FailOnTimeoutGroup] 2024-12-01T14:55:18,848 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/519b220a2ba1:0:becomeActiveMaster-HFileCleaner.small.0-1733064918848,5,FailOnTimeoutGroup] 2024-12-01T14:55:18,848 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,848 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-01T14:55:18,848 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,848 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,857 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-01T14:55:18,857 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46 2024-12-01T14:55:18,863 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:55:18,864 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-01T14:55:18,868 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.HRegionServer(746): ClusterId : bcc9be81-f2fd-46ef-ab98-7c66ad66ea6c 2024-12-01T14:55:18,868 INFO [RS:0;519b220a2ba1:34279 {}] regionserver.HRegionServer(746): ClusterId : bcc9be81-f2fd-46ef-ab98-7c66ad66ea6c 2024-12-01T14:55:18,868 DEBUG [RS:1;519b220a2ba1:40943 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T14:55:18,868 DEBUG [RS:0;519b220a2ba1:34279 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T14:55:18,870 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T14:55:18,870 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:18,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:18,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-01T14:55:18,874 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T14:55:18,874 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:18,874 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:18,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-01T14:55:18,878 DEBUG [RS:1;519b220a2ba1:40943 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T14:55:18,878 DEBUG [RS:1;519b220a2ba1:40943 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T14:55:18,878 DEBUG [RS:0;519b220a2ba1:34279 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T14:55:18,878 DEBUG [RS:0;519b220a2ba1:34279 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T14:55:18,879 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T14:55:18,879 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:18,880 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:18,880 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-01T14:55:18,884 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T14:55:18,884 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:18,884 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:18,885 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T14:55:18,885 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740 2024-12-01T14:55:18,885 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740 2024-12-01T14:55:18,886 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T14:55:18,886 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T14:55:18,886 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T14:55:18,886 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T14:55:18,895 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T14:55:18,895 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72433584, jitterRate=0.07934451103210449}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T14:55:18,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733064918863Initializing all the Stores at 1733064918863Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733064918863Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733064918864 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733064918864Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733064918864Cleaning up temporary data from old regions at 1733064918886 (+22 ms)Region opened successfully at 1733064918895 (+9 ms) 2024-12-01T14:55:18,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T14:55:18,896 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T14:55:18,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T14:55:18,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T14:55:18,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T14:55:18,896 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T14:55:18,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733064918896Disabling compacts and flushes for region at 1733064918896Disabling writes for close at 1733064918896Writing region close event to WAL at 1733064918896Closed at 1733064918896 2024-12-01T14:55:18,897 DEBUG [RS:1;519b220a2ba1:40943 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T14:55:18,897 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T14:55:18,897 DEBUG [RS:0;519b220a2ba1:34279 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T14:55:18,897 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-01T14:55:18,897 DEBUG [RS:1;519b220a2ba1:40943 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2788694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=519b220a2ba1/172.17.0.2:0 2024-12-01T14:55:18,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-01T14:55:18,897 DEBUG [RS:0;519b220a2ba1:34279 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@412c739c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=519b220a2ba1/172.17.0.2:0 2024-12-01T14:55:18,899 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T14:55:18,899 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-01T14:55:18,909 DEBUG [RS:1;519b220a2ba1:40943 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;519b220a2ba1:40943 2024-12-01T14:55:18,909 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T14:55:18,909 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T14:55:18,909 DEBUG [RS:1;519b220a2ba1:40943 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T14:55:18,909 DEBUG [RS:0;519b220a2ba1:34279 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;519b220a2ba1:34279 2024-12-01T14:55:18,909 INFO [RS:0;519b220a2ba1:34279 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T14:55:18,909 INFO [RS:0;519b220a2ba1:34279 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T14:55:18,909 DEBUG [RS:0;519b220a2ba1:34279 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T14:55:18,910 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.HRegionServer(2659): reportForDuty to master=519b220a2ba1,42737,1733064918246 with port=40943, startcode=1733064918444 2024-12-01T14:55:18,910 INFO [RS:0;519b220a2ba1:34279 {}] regionserver.HRegionServer(2659): reportForDuty to master=519b220a2ba1,42737,1733064918246 with port=34279, startcode=1733064918408 2024-12-01T14:55:18,910 DEBUG [RS:1;519b220a2ba1:40943 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T14:55:18,910 DEBUG [RS:0;519b220a2ba1:34279 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T14:55:18,913 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46269, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T14:55:18,913 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46503, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T14:55:18,913 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42737 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 519b220a2ba1,34279,1733064918408 2024-12-01T14:55:18,914 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42737 {}] master.ServerManager(517): Registering regionserver=519b220a2ba1,34279,1733064918408 2024-12-01T14:55:18,915 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42737 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 519b220a2ba1,40943,1733064918444 2024-12-01T14:55:18,915 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42737 {}] master.ServerManager(517): Registering regionserver=519b220a2ba1,40943,1733064918444 2024-12-01T14:55:18,915 DEBUG [RS:0;519b220a2ba1:34279 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46 2024-12-01T14:55:18,915 DEBUG [RS:0;519b220a2ba1:34279 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-01T14:55:18,915 DEBUG [RS:0;519b220a2ba1:34279 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-01T14:55:18,917 DEBUG [RS:1;519b220a2ba1:40943 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46 2024-12-01T14:55:18,917 DEBUG [RS:1;519b220a2ba1:40943 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-01T14:55:18,917 DEBUG [RS:1;519b220a2ba1:40943 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-01T14:55:18,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T14:55:18,959 DEBUG [RS:0;519b220a2ba1:34279 {}] zookeeper.ZKUtil(111): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/519b220a2ba1,34279,1733064918408 2024-12-01T14:55:18,959 WARN [RS:0;519b220a2ba1:34279 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T14:55:18,959 INFO [RS:0;519b220a2ba1:34279 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:55:18,959 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [519b220a2ba1,34279,1733064918408] 2024-12-01T14:55:18,959 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [519b220a2ba1,40943,1733064918444] 2024-12-01T14:55:18,959 DEBUG [RS:0;519b220a2ba1:34279 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,34279,1733064918408 2024-12-01T14:55:18,959 DEBUG [RS:1;519b220a2ba1:40943 {}] zookeeper.ZKUtil(111): regionserver:40943-0x10192ad978f0002, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/519b220a2ba1,40943,1733064918444 2024-12-01T14:55:18,959 WARN [RS:1;519b220a2ba1:40943 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T14:55:18,959 INFO [RS:1;519b220a2ba1:40943 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:55:18,960 DEBUG [RS:1;519b220a2ba1:40943 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,40943,1733064918444 2024-12-01T14:55:18,968 INFO [RS:0;519b220a2ba1:34279 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T14:55:18,968 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T14:55:18,971 INFO [RS:0;519b220a2ba1:34279 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T14:55:18,971 INFO [RS:0;519b220a2ba1:34279 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T14:55:18,971 INFO [RS:0;519b220a2ba1:34279 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,972 INFO [RS:0;519b220a2ba1:34279 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T14:55:18,972 INFO [RS:0;519b220a2ba1:34279 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T14:55:18,972 INFO [RS:0;519b220a2ba1:34279 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,972 DEBUG [RS:0;519b220a2ba1:34279 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,972 DEBUG [RS:0;519b220a2ba1:34279 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,972 DEBUG [RS:0;519b220a2ba1:34279 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,972 DEBUG [RS:0;519b220a2ba1:34279 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,973 DEBUG [RS:0;519b220a2ba1:34279 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,973 DEBUG [RS:0;519b220a2ba1:34279 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/519b220a2ba1:0, corePoolSize=2, maxPoolSize=2 2024-12-01T14:55:18,973 DEBUG [RS:0;519b220a2ba1:34279 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,973 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T14:55:18,973 DEBUG [RS:0;519b220a2ba1:34279 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,973 DEBUG [RS:0;519b220a2ba1:34279 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,973 DEBUG [RS:0;519b220a2ba1:34279 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,973 DEBUG [RS:0;519b220a2ba1:34279 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,973 DEBUG [RS:0;519b220a2ba1:34279 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,973 INFO [RS:1;519b220a2ba1:40943 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T14:55:18,973 DEBUG [RS:0;519b220a2ba1:34279 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/519b220a2ba1:0, corePoolSize=3, maxPoolSize=3 2024-12-01T14:55:18,973 DEBUG [RS:0;519b220a2ba1:34279 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/519b220a2ba1:0, corePoolSize=3, maxPoolSize=3 2024-12-01T14:55:18,973 INFO [RS:1;519b220a2ba1:40943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,974 INFO [RS:0;519b220a2ba1:34279 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,974 INFO [RS:0;519b220a2ba1:34279 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,974 INFO [RS:0;519b220a2ba1:34279 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,974 INFO [RS:0;519b220a2ba1:34279 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,974 INFO [RS:0;519b220a2ba1:34279 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,974 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T14:55:18,974 INFO [RS:0;519b220a2ba1:34279 {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,34279,1733064918408-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T14:55:18,975 INFO [RS:1;519b220a2ba1:40943 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T14:55:18,975 INFO [RS:1;519b220a2ba1:40943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,975 DEBUG [RS:1;519b220a2ba1:40943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,975 DEBUG [RS:1;519b220a2ba1:40943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,975 DEBUG [RS:1;519b220a2ba1:40943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,975 DEBUG [RS:1;519b220a2ba1:40943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,975 DEBUG [RS:1;519b220a2ba1:40943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,976 DEBUG [RS:1;519b220a2ba1:40943 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/519b220a2ba1:0, corePoolSize=2, maxPoolSize=2 2024-12-01T14:55:18,976 DEBUG [RS:1;519b220a2ba1:40943 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,976 DEBUG [RS:1;519b220a2ba1:40943 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,976 DEBUG [RS:1;519b220a2ba1:40943 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,976 DEBUG [RS:1;519b220a2ba1:40943 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,976 DEBUG [RS:1;519b220a2ba1:40943 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,976 DEBUG [RS:1;519b220a2ba1:40943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:18,976 DEBUG [RS:1;519b220a2ba1:40943 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/519b220a2ba1:0, corePoolSize=3, maxPoolSize=3 2024-12-01T14:55:18,976 DEBUG [RS:1;519b220a2ba1:40943 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/519b220a2ba1:0, corePoolSize=3, maxPoolSize=3 2024-12-01T14:55:18,977 INFO [RS:1;519b220a2ba1:40943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,978 INFO [RS:1;519b220a2ba1:40943 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,978 INFO [RS:1;519b220a2ba1:40943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,978 INFO [RS:1;519b220a2ba1:40943 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,978 INFO [RS:1;519b220a2ba1:40943 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,978 INFO [RS:1;519b220a2ba1:40943 {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,40943,1733064918444-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T14:55:18,987 INFO [RS:0;519b220a2ba1:34279 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T14:55:18,987 INFO [RS:0;519b220a2ba1:34279 {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,34279,1733064918408-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,987 INFO [RS:0;519b220a2ba1:34279 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,987 INFO [RS:0;519b220a2ba1:34279 {}] regionserver.Replication(171): 519b220a2ba1,34279,1733064918408 started 2024-12-01T14:55:18,989 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T14:55:18,989 INFO [RS:1;519b220a2ba1:40943 {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,40943,1733064918444-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,989 INFO [RS:1;519b220a2ba1:40943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:18,990 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.Replication(171): 519b220a2ba1,40943,1733064918444 started 2024-12-01T14:55:19,000 INFO [RS:0;519b220a2ba1:34279 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:19,000 INFO [RS:0;519b220a2ba1:34279 {}] regionserver.HRegionServer(1482): Serving as 519b220a2ba1,34279,1733064918408, RpcServer on 519b220a2ba1/172.17.0.2:34279, sessionid=0x10192ad978f0001 2024-12-01T14:55:19,000 DEBUG [RS:0;519b220a2ba1:34279 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T14:55:19,000 DEBUG [RS:0;519b220a2ba1:34279 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 519b220a2ba1,34279,1733064918408 2024-12-01T14:55:19,000 DEBUG [RS:0;519b220a2ba1:34279 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '519b220a2ba1,34279,1733064918408' 2024-12-01T14:55:19,000 DEBUG [RS:0;519b220a2ba1:34279 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-01T14:55:19,001 DEBUG [RS:0;519b220a2ba1:34279 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-01T14:55:19,001 DEBUG [RS:0;519b220a2ba1:34279 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T14:55:19,001 DEBUG [RS:0;519b220a2ba1:34279 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T14:55:19,001 DEBUG [RS:0;519b220a2ba1:34279 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 519b220a2ba1,34279,1733064918408 2024-12-01T14:55:19,002 DEBUG [RS:0;519b220a2ba1:34279 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '519b220a2ba1,34279,1733064918408' 2024-12-01T14:55:19,002 DEBUG [RS:0;519b220a2ba1:34279 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-01T14:55:19,002 DEBUG [RS:0;519b220a2ba1:34279 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-01T14:55:19,002 DEBUG [RS:0;519b220a2ba1:34279 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T14:55:19,003 INFO [RS:0;519b220a2ba1:34279 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T14:55:19,003 INFO [RS:0;519b220a2ba1:34279 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T14:55:19,003 INFO [RS:1;519b220a2ba1:40943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:19,003 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.HRegionServer(1482): Serving as 519b220a2ba1,40943,1733064918444, RpcServer on 519b220a2ba1/172.17.0.2:40943, sessionid=0x10192ad978f0002 2024-12-01T14:55:19,004 DEBUG [RS:1;519b220a2ba1:40943 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T14:55:19,004 DEBUG [RS:1;519b220a2ba1:40943 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 519b220a2ba1,40943,1733064918444 2024-12-01T14:55:19,004 DEBUG [RS:1;519b220a2ba1:40943 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '519b220a2ba1,40943,1733064918444' 2024-12-01T14:55:19,004 DEBUG [RS:1;519b220a2ba1:40943 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-01T14:55:19,004 DEBUG [RS:1;519b220a2ba1:40943 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-01T14:55:19,005 DEBUG [RS:1;519b220a2ba1:40943 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T14:55:19,005 DEBUG [RS:1;519b220a2ba1:40943 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T14:55:19,005 DEBUG [RS:1;519b220a2ba1:40943 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 519b220a2ba1,40943,1733064918444 2024-12-01T14:55:19,005 DEBUG [RS:1;519b220a2ba1:40943 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '519b220a2ba1,40943,1733064918444' 2024-12-01T14:55:19,005 DEBUG [RS:1;519b220a2ba1:40943 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-01T14:55:19,005 DEBUG [RS:1;519b220a2ba1:40943 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-01T14:55:19,006 DEBUG [RS:1;519b220a2ba1:40943 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T14:55:19,006 INFO [RS:1;519b220a2ba1:40943 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T14:55:19,006 INFO [RS:1;519b220a2ba1:40943 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T14:55:19,050 WARN [519b220a2ba1:42737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T14:55:19,104 INFO [RS:0;519b220a2ba1:34279 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:55:19,105 INFO [RS:0;519b220a2ba1:34279 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=519b220a2ba1%2C34279%2C1733064918408, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,34279,1733064918408, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/oldWALs, maxLogs=55 2024-12-01T14:55:19,107 INFO [RS:1;519b220a2ba1:40943 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:55:19,108 INFO [RS:1;519b220a2ba1:40943 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=519b220a2ba1%2C40943%2C1733064918444, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,40943,1733064918444, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/oldWALs, maxLogs=55 2024-12-01T14:55:19,126 INFO [RS:0;519b220a2ba1:34279 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,34279,1733064918408/519b220a2ba1%2C34279%2C1733064918408.1733064919110 2024-12-01T14:55:19,127 DEBUG [RS:0;519b220a2ba1:34279 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-01T14:55:19,127 INFO [RS:1;519b220a2ba1:40943 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,40943,1733064918444/519b220a2ba1%2C40943%2C1733064918444.1733064919110 2024-12-01T14:55:19,128 DEBUG [RS:1;519b220a2ba1:40943 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-01T14:55:19,300 DEBUG [519b220a2ba1:42737 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-01T14:55:19,300 DEBUG [519b220a2ba1:42737 {}] balancer.BalancerClusterState(204): Hosts are {519b220a2ba1=0} racks are {/default-rack=0} 2024-12-01T14:55:19,301 DEBUG [519b220a2ba1:42737 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T14:55:19,301 DEBUG [519b220a2ba1:42737 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T14:55:19,301 DEBUG [519b220a2ba1:42737 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T14:55:19,301 DEBUG [519b220a2ba1:42737 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T14:55:19,301 INFO [519b220a2ba1:42737 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T14:55:19,301 INFO [519b220a2ba1:42737 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T14:55:19,301 DEBUG [519b220a2ba1:42737 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T14:55:19,302 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=519b220a2ba1,40943,1733064918444 2024-12-01T14:55:19,303 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 519b220a2ba1,40943,1733064918444, state=OPENING 2024-12-01T14:55:19,338 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-01T14:55:19,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40943-0x10192ad978f0002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:19,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:19,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:19,347 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:55:19,347 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T14:55:19,347 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:55:19,347 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:55:19,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=519b220a2ba1,40943,1733064918444}] 2024-12-01T14:55:19,500 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T14:55:19,502 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43851, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T14:55:19,505 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-01T14:55:19,505 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:55:19,506 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-01T14:55:19,507 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=519b220a2ba1%2C40943%2C1733064918444.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,40943,1733064918444, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/oldWALs, maxLogs=55 2024-12-01T14:55:19,531 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,40943,1733064918444/519b220a2ba1%2C40943%2C1733064918444.meta.1733064919508.meta 2024-12-01T14:55:19,532 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-01T14:55:19,532 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:55:19,532 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-01T14:55:19,533 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-01T14:55:19,533 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-01T14:55:19,533 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-01T14:55:19,533 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:55:19,533 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-01T14:55:19,533 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-01T14:55:19,534 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-01T14:55:19,535 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T14:55:19,535 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:19,535 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:19,535 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-01T14:55:19,536 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T14:55:19,536 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:19,536 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:19,536 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-01T14:55:19,536 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T14:55:19,536 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:19,537 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:19,537 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-01T14:55:19,537 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T14:55:19,537 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:19,538 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:19,538 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T14:55:19,539 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740 2024-12-01T14:55:19,539 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740 2024-12-01T14:55:19,539 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T14:55:19,540 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T14:55:19,540 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T14:55:19,541 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T14:55:19,541 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67572558, jitterRate=0.006909579038619995}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T14:55:19,541 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-01T14:55:19,542 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733064919534Writing region info on filesystem at 1733064919534Initializing all the Stores at 1733064919534Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733064919534Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733064919534Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733064919534Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733064919534Cleaning up temporary data from old regions at 1733064919540 (+6 ms)Running coprocessor post-open hooks at 1733064919541 (+1 ms)Region opened successfully at 1733064919542 (+1 ms) 2024-12-01T14:55:19,543 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733064919500 2024-12-01T14:55:19,546 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-01T14:55:19,546 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-01T14:55:19,546 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=519b220a2ba1,40943,1733064918444 2024-12-01T14:55:19,547 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 519b220a2ba1,40943,1733064918444, state=OPEN 2024-12-01T14:55:19,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T14:55:19,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40943-0x10192ad978f0002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T14:55:19,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T14:55:19,571 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=519b220a2ba1,40943,1733064918444 2024-12-01T14:55:19,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:55:19,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:55:19,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:55:19,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-01T14:55:19,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=519b220a2ba1,40943,1733064918444 in 225 msec 2024-12-01T14:55:19,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-01T14:55:19,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 677 msec 2024-12-01T14:55:19,576 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T14:55:19,576 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-01T14:55:19,579 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T14:55:19,579 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=519b220a2ba1,40943,1733064918444, seqNum=-1] 2024-12-01T14:55:19,580 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T14:55:19,581 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54271, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T14:55:19,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 744 msec 2024-12-01T14:55:19,586 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733064919586, completionTime=-1 2024-12-01T14:55:19,586 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-01T14:55:19,587 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-01T14:55:19,589 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-01T14:55:19,589 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733064979589 2024-12-01T14:55:19,589 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733065039589 2024-12-01T14:55:19,589 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-01T14:55:19,590 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,42737,1733064918246-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:19,590 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,42737,1733064918246-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:19,590 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,42737,1733064918246-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:19,590 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-519b220a2ba1:42737, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:19,590 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:19,590 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:19,593 DEBUG [master/519b220a2ba1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-01T14:55:19,596 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.091sec 2024-12-01T14:55:19,596 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-01T14:55:19,596 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-01T14:55:19,596 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-01T14:55:19,596 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-01T14:55:19,596 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-01T14:55:19,596 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,42737,1733064918246-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T14:55:19,596 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,42737,1733064918246-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-01T14:55:19,597 DEBUG [master/519b220a2ba1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-01T14:55:19,597 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-01T14:55:19,597 INFO [master/519b220a2ba1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,42737,1733064918246-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:19,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b03a5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T14:55:19,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 519b220a2ba1,42737,-1 for getting cluster id 2024-12-01T14:55:19,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T14:55:19,669 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bcc9be81-f2fd-46ef-ab98-7c66ad66ea6c' 2024-12-01T14:55:19,670 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T14:55:19,670 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bcc9be81-f2fd-46ef-ab98-7c66ad66ea6c" 2024-12-01T14:55:19,670 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3881ec6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T14:55:19,670 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [519b220a2ba1,42737,-1] 2024-12-01T14:55:19,671 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T14:55:19,671 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:55:19,673 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T14:55:19,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42ddb663, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T14:55:19,675 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T14:55:19,676 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=519b220a2ba1,40943,1733064918444, seqNum=-1] 2024-12-01T14:55:19,677 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T14:55:19,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T14:55:19,681 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=519b220a2ba1,42737,1733064918246 2024-12-01T14:55:19,681 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-01T14:55:19,682 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 519b220a2ba1,42737,1733064918246 2024-12-01T14:55:19,683 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@577968d9 2024-12-01T14:55:19,683 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-01T14:55:19,685 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-01T14:55:19,686 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42737 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733064919681', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T14:55:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42737 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733064919681 2024-12-01T14:55:19,688 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733064919681 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T14:55:19,688 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:19,688 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42737 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733064919681" procId is: 4 2024-12-01T14:55:19,689 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733064919681 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T14:55:19,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T14:55:19,698 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733064919681-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5754fb1c74471df5775e3ac5ebf5eb44, NAME => 'testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733064919681', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46 2024-12-01T14:55:19,704 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733064919681-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:55:19,704 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733064919681-pool-0 {}] regionserver.HRegion(1722): Closing 5754fb1c74471df5775e3ac5ebf5eb44, disabling compactions & flushes 2024-12-01T14:55:19,704 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733064919681-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44. 2024-12-01T14:55:19,704 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733064919681-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44. 2024-12-01T14:55:19,704 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733064919681-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44. after waiting 0 ms 2024-12-01T14:55:19,704 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733064919681-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44. 2024-12-01T14:55:19,704 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733064919681-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44. 2024-12-01T14:55:19,704 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733064919681-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5754fb1c74471df5775e3ac5ebf5eb44: Waiting for close lock at 1733064919704Disabling compacts and flushes for region at 1733064919704Disabling writes for close at 1733064919704Writing region close event to WAL at 1733064919704Closed at 1733064919704 2024-12-01T14:55:19,706 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733064919681 execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T14:55:19,706 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733064919706"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733064919706"}]},"ts":"1733064919706"} 2024-12-01T14:55:19,708 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T14:55:19,709 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733064919681 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T14:55:19,710 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733064919681","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733064919709"}]},"ts":"1733064919709"} 2024-12-01T14:55:19,712 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733064919681, state=ENABLING in hbase:meta 2024-12-01T14:55:19,712 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {519b220a2ba1=0} racks are {/default-rack=0} 2024-12-01T14:55:19,712 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T14:55:19,712 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T14:55:19,712 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T14:55:19,712 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T14:55:19,713 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T14:55:19,713 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T14:55:19,713 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T14:55:19,713 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733064919681, region=5754fb1c74471df5775e3ac5ebf5eb44, ASSIGN}] 2024-12-01T14:55:19,714 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733064919681, region=5754fb1c74471df5775e3ac5ebf5eb44, ASSIGN 2024-12-01T14:55:19,715 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733064919681, region=5754fb1c74471df5775e3ac5ebf5eb44, ASSIGN; state=OFFLINE, location=519b220a2ba1,40943,1733064918444; forceNewPlan=false, retain=false 2024-12-01T14:55:19,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T14:55:19,865 INFO [519b220a2ba1:42737 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-01T14:55:19,866 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5754fb1c74471df5775e3ac5ebf5eb44, regionState=OPENING, regionLocation=519b220a2ba1,40943,1733064918444 2024-12-01T14:55:19,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733064919681, region=5754fb1c74471df5775e3ac5ebf5eb44, ASSIGN because future has completed 2024-12-01T14:55:19,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5754fb1c74471df5775e3ac5ebf5eb44, server=519b220a2ba1,40943,1733064918444}] 2024-12-01T14:55:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T14:55:20,024 INFO [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44. 2024-12-01T14:55:20,024 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5754fb1c74471df5775e3ac5ebf5eb44, NAME => 'testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:55:20,025 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733064919681 5754fb1c74471df5775e3ac5ebf5eb44 2024-12-01T14:55:20,025 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:55:20,025 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5754fb1c74471df5775e3ac5ebf5eb44 2024-12-01T14:55:20,025 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5754fb1c74471df5775e3ac5ebf5eb44 2024-12-01T14:55:20,025 INFO [StoreOpener-5754fb1c74471df5775e3ac5ebf5eb44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 5754fb1c74471df5775e3ac5ebf5eb44 2024-12-01T14:55:20,028 INFO [StoreOpener-5754fb1c74471df5775e3ac5ebf5eb44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5754fb1c74471df5775e3ac5ebf5eb44 columnFamilyName family 2024-12-01T14:55:20,029 DEBUG [StoreOpener-5754fb1c74471df5775e3ac5ebf5eb44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:20,029 INFO [StoreOpener-5754fb1c74471df5775e3ac5ebf5eb44-1 {}] regionserver.HStore(327): Store=5754fb1c74471df5775e3ac5ebf5eb44/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:55:20,029 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5754fb1c74471df5775e3ac5ebf5eb44 2024-12-01T14:55:20,029 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnGracefulStop1733064919681/5754fb1c74471df5775e3ac5ebf5eb44 2024-12-01T14:55:20,030 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnGracefulStop1733064919681/5754fb1c74471df5775e3ac5ebf5eb44 2024-12-01T14:55:20,030 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5754fb1c74471df5775e3ac5ebf5eb44 2024-12-01T14:55:20,030 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5754fb1c74471df5775e3ac5ebf5eb44 2024-12-01T14:55:20,030 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5754fb1c74471df5775e3ac5ebf5eb44 2024-12-01T14:55:20,035 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnGracefulStop1733064919681/5754fb1c74471df5775e3ac5ebf5eb44/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T14:55:20,036 INFO [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5754fb1c74471df5775e3ac5ebf5eb44; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66415267, jitterRate=-0.010335400700569153}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T14:55:20,036 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5754fb1c74471df5775e3ac5ebf5eb44 2024-12-01T14:55:20,036 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5754fb1c74471df5775e3ac5ebf5eb44: Running coprocessor pre-open hook at 1733064920025Writing region info on filesystem at 1733064920025Initializing all the Stores at 1733064920025Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733064920025Cleaning up temporary data from old regions at 1733064920030 (+5 ms)Running coprocessor post-open hooks at 1733064920036 (+6 ms)Region opened successfully at 1733064920036 2024-12-01T14:55:20,037 INFO [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44., pid=6, masterSystemTime=1733064920022 2024-12-01T14:55:20,040 DEBUG [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44. 2024-12-01T14:55:20,040 INFO [RS_OPEN_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44. 2024-12-01T14:55:20,040 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5754fb1c74471df5775e3ac5ebf5eb44, regionState=OPEN, openSeqNum=2, regionLocation=519b220a2ba1,40943,1733064918444 2024-12-01T14:55:20,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5754fb1c74471df5775e3ac5ebf5eb44, server=519b220a2ba1,40943,1733064918444 because future has completed 2024-12-01T14:55:20,047 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-01T14:55:20,047 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5754fb1c74471df5775e3ac5ebf5eb44, server=519b220a2ba1,40943,1733064918444 in 176 msec 2024-12-01T14:55:20,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-01T14:55:20,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733064919681, region=5754fb1c74471df5775e3ac5ebf5eb44, ASSIGN in 334 msec 2024-12-01T14:55:20,050 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733064919681 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T14:55:20,050 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733064919681","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733064920050"}]},"ts":"1733064920050"} 2024-12-01T14:55:20,053 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733064919681, state=ENABLED in hbase:meta 2024-12-01T14:55:20,054 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733064919681 execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T14:55:20,055 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733064919681 in 368 msec 2024-12-01T14:55:20,249 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T14:55:20,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T14:55:20,314 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733064919681 completed 2024-12-01T14:55:20,314 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733064919681 get assigned. Timeout = 60000ms 2024-12-01T14:55:20,314 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T14:55:20,318 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733064919681 assigned to meta. Checking AM states. 2024-12-01T14:55:20,318 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T14:55:20,318 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733064919681 assigned. 2024-12-01T14:55:20,323 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733064919681', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44., hostname=519b220a2ba1,40943,1733064918444, seqNum=2] 2024-12-01T14:55:20,329 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5754fb1c74471df5775e3ac5ebf5eb44 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-01T14:55:20,338 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5c399765e52d4d2b93385c33113093ca 2024-12-01T14:55:20,351 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnGracefulStop1733064919681/5754fb1c74471df5775e3ac5ebf5eb44/.tmp/family/5c399765e52d4d2b93385c33113093ca is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-01T14:55:20,352 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnGracefulStop1733064919681/5754fb1c74471df5775e3ac5ebf5eb44/.tmp/family/5c399765e52d4d2b93385c33113093ca 2024-12-01T14:55:20,355 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnGracefulStop1733064919681/5754fb1c74471df5775e3ac5ebf5eb44/.tmp/family/5c399765e52d4d2b93385c33113093ca as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnGracefulStop1733064919681/5754fb1c74471df5775e3ac5ebf5eb44/family/5c399765e52d4d2b93385c33113093ca 2024-12-01T14:55:20,356 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnGracefulStop1733064919681/5754fb1c74471df5775e3ac5ebf5eb44/family/5c399765e52d4d2b93385c33113093ca, delay=990 ms 2024-12-01T14:55:20,358 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnGracefulStop1733064919681/5754fb1c74471df5775e3ac5ebf5eb44/family/5c399765e52d4d2b93385c33113093ca, entries=2, sequenceid=6, filesize=4.9 K 2024-12-01T14:55:20,358 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 5754fb1c74471df5775e3ac5ebf5eb44 in 29ms, sequenceid=6, compaction requested=false 2024-12-01T14:55:20,358 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733064919681' 2024-12-01T14:55:20,359 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5754fb1c74471df5775e3ac5ebf5eb44: 2024-12-01T14:55:21,348 DEBUG [hfile-prefetch-1733064916000 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 5c399765e52d4d2b93385c33113093ca_0. 2024-12-01T14:55:21,349 DEBUG [hfile-prefetch-1733064916000 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 5c399765e52d4d2b93385c33113093ca_117. 2024-12-01T14:55:21,349 DEBUG [hfile-prefetch-1733064916000 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnGracefulStop1733064919681/5754fb1c74471df5775e3ac5ebf5eb44/family/5c399765e52d4d2b93385c33113093ca, with total blocks 2, and data blocks 1 2024-12-01T14:55:21,349 DEBUG [hfile-prefetch-1733064916000 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-01T14:55:21,349 DEBUG [hfile-prefetch-1733064916000 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnGracefulStop1733064919681/5754fb1c74471df5775e3ac5ebf5eb44/family/5c399765e52d4d2b93385c33113093ca, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-01T14:55:21,349 DEBUG [hfile-prefetch-1733064916000 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnGracefulStop1733064919681/5754fb1c74471df5775e3ac5ebf5eb44/family/5c399765e52d4d2b93385c33113093ca. Will sleep 100ms and try the verification again. 2024-12-01T14:55:21,359 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;519b220a2ba1:40943,5,FailOnTimeoutGroup] 2024-12-01T14:55:21,360 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '519b220a2ba1,40943,1733064918444' ***** 2024-12-01T14:55:21,360 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-12-01T14:55:21,360 INFO [RS:1;519b220a2ba1:40943 {}] bucket.BucketCache(1738): Disabling cache 2024-12-01T14:55:21,380 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T14:55:21,380 INFO [regionserver/519b220a2ba1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T14:55:21,426 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-01T14:55:21,426 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-01T14:55:21,426 INFO [RS:1;519b220a2ba1:40943 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1/bucket.persistence 2024-12-01T14:55:21,426 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-01T14:55:21,426 INFO [RS:1;519b220a2ba1:40943 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-01T14:55:21,426 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-01T14:55:21,426 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-01T14:55:21,426 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-01T14:55:21,426 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-01T14:55:21,426 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-01T14:55:21,429 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-01T14:55:21,430 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733064921426 2024-12-01T14:55:21,430 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-01T14:55:21,436 DEBUG [RS:1;519b220a2ba1:40943 {}] bucket.BucketCache(1487): Thread RS:1;519b220a2ba1:40943 started persisting bucket cache to file 2024-12-01T14:55:21,436 DEBUG [RS:1;519b220a2ba1:40943 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-01T14:55:21,436 DEBUG [RS:1;519b220a2ba1:40943 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-01T14:55:21,446 DEBUG [RS:1;519b220a2ba1:40943 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-01T14:55:21,447 DEBUG [RS:1;519b220a2ba1:40943 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733064921436 2024-12-01T14:55:21,447 DEBUG [RS:1;519b220a2ba1:40943 {}] bucket.BucketCache(1509): Thread RS:1;519b220a2ba1:40943 finished persisting bucket cache to file, renaming 2024-12-01T14:55:21,447 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T14:55:21,447 INFO [RS:1;519b220a2ba1:40943 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-01T14:55:21,447 INFO [RS:1;519b220a2ba1:40943 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-01T14:55:21,447 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.HRegionServer(3091): Received CLOSE for 5754fb1c74471df5775e3ac5ebf5eb44 2024-12-01T14:55:21,448 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.HRegionServer(959): stopping server 519b220a2ba1,40943,1733064918444 2024-12-01T14:55:21,448 INFO [RS:1;519b220a2ba1:40943 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T14:55:21,448 INFO [RS:1;519b220a2ba1:40943 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;519b220a2ba1:40943. 2024-12-01T14:55:21,448 INFO [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-01T14:55:21,448 DEBUG [RS:1;519b220a2ba1:40943 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T14:55:21,448 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5754fb1c74471df5775e3ac5ebf5eb44, disabling compactions & flushes 2024-12-01T14:55:21,449 DEBUG [RS:1;519b220a2ba1:40943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:55:21,449 INFO [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44. 2024-12-01T14:55:21,449 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T14:55:21,449 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44. 2024-12-01T14:55:21,449 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T14:55:21,449 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T14:55:21,449 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44. after waiting 0 ms 2024-12-01T14:55:21,449 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-01T14:55:21,449 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44. 2024-12-01T14:55:21,449 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-01T14:55:21,450 DEBUG [RS:1;519b220a2ba1:40943 {}] regionserver.HRegionServer(1325): Online Regions={5754fb1c74471df5775e3ac5ebf5eb44=testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44., 1588230740=hbase:meta,,1.1588230740} 2024-12-01T14:55:21,450 DEBUG [hfile-prefetch-1733064916000 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnGracefulStop1733064919681/5754fb1c74471df5775e3ac5ebf5eb44/family/5c399765e52d4d2b93385c33113093ca, with total blocks 2, and data blocks 1 2024-12-01T14:55:21,450 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-01T14:55:21,450 DEBUG [hfile-prefetch-1733064916000 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-12-01T14:55:21,450 DEBUG [RS:1;519b220a2ba1:40943 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 5754fb1c74471df5775e3ac5ebf5eb44 2024-12-01T14:55:21,450 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T14:55:21,450 DEBUG [hfile-prefetch-1733064916000 {}] bucket.BucketCache(2293): found block for file 5c399765e52d4d2b93385c33113093ca in the backing map. Acquiring read lock for offset 0 2024-12-01T14:55:21,450 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T14:55:21,450 DEBUG [hfile-prefetch-1733064916000 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnGracefulStop1733064919681/5754fb1c74471df5775e3ac5ebf5eb44/family/5c399765e52d4d2b93385c33113093ca has now been fully cached. 2024-12-01T14:55:21,450 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T14:55:21,450 DEBUG [StoreFileCloser-5754fb1c74471df5775e3ac5ebf5eb44-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnGracefulStop1733064919681/5754fb1c74471df5775e3ac5ebf5eb44/family/5c399765e52d4d2b93385c33113093ca 2024-12-01T14:55:21,450 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T14:55:21,450 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T14:55:21,451 DEBUG [hfile-prefetch-1733064916000 {}] hfile.PrefetchExecutor(121): Prefetch completed for 5c399765e52d4d2b93385c33113093ca 2024-12-01T14:55:21,451 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-12-01T14:55:21,456 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/default/testBlockEvictionOnGracefulStop1733064919681/5754fb1c74471df5775e3ac5ebf5eb44/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-01T14:55:21,456 INFO [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44. 2024-12-01T14:55:21,456 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5754fb1c74471df5775e3ac5ebf5eb44: Waiting for close lock at 1733064921448Running coprocessor pre-close hooks at 1733064921448Disabling compacts and flushes for region at 1733064921448Disabling writes for close at 1733064921449 (+1 ms)Writing region close event to WAL at 1733064921451 (+2 ms)Running coprocessor post-close hooks at 1733064921456 (+5 ms)Closed at 1733064921456 2024-12-01T14:55:21,457 DEBUG [RS_CLOSE_REGION-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44. 2024-12-01T14:55:21,461 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 695bc492c785402fa5a03f41534ca71f 2024-12-01T14:55:21,471 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/info/695bc492c785402fa5a03f41534ca71f is 209, key is testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44./info:regioninfo/1733064920040/Put/seqid=0 2024-12-01T14:55:21,472 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/info/695bc492c785402fa5a03f41534ca71f 2024-12-01T14:55:21,479 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c304f3e7733749de870693ee9a745391 2024-12-01T14:55:21,490 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/ns/c304f3e7733749de870693ee9a745391 is 43, key is default/ns:d/1733064919582/Put/seqid=0 2024-12-01T14:55:21,491 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/ns/c304f3e7733749de870693ee9a745391 2024-12-01T14:55:21,499 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a362e02f68ca4932bbe9dcf0fbf2cd7b 2024-12-01T14:55:21,499 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-01T14:55:21,499 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-01T14:55:21,501 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T14:55:21,501 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-01T14:55:21,501 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-01T14:55:21,501 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-01T14:55:21,510 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/table/a362e02f68ca4932bbe9dcf0fbf2cd7b is 80, key is testBlockEvictionOnGracefulStop1733064919681/table:state/1733064920050/Put/seqid=0 2024-12-01T14:55:21,511 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/table/a362e02f68ca4932bbe9dcf0fbf2cd7b 2024-12-01T14:55:21,514 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/info/695bc492c785402fa5a03f41534ca71f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/info/695bc492c785402fa5a03f41534ca71f 2024-12-01T14:55:21,515 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/info/695bc492c785402fa5a03f41534ca71f, delay=973 ms 2024-12-01T14:55:21,517 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/info/695bc492c785402fa5a03f41534ca71f, entries=10, sequenceid=11, filesize=7.0 K 2024-12-01T14:55:21,517 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/ns/c304f3e7733749de870693ee9a745391 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/ns/c304f3e7733749de870693ee9a745391 2024-12-01T14:55:21,519 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/ns/c304f3e7733749de870693ee9a745391, delay=974 ms 2024-12-01T14:55:21,520 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/ns/c304f3e7733749de870693ee9a745391, entries=2, sequenceid=11, filesize=5.0 K 2024-12-01T14:55:21,520 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/table/a362e02f68ca4932bbe9dcf0fbf2cd7b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/table/a362e02f68ca4932bbe9dcf0fbf2cd7b 2024-12-01T14:55:21,521 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/table/a362e02f68ca4932bbe9dcf0fbf2cd7b, delay=925 ms 2024-12-01T14:55:21,522 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/table/a362e02f68ca4932bbe9dcf0fbf2cd7b, entries=2, sequenceid=11, filesize=5.3 K 2024-12-01T14:55:21,523 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 72ms, sequenceid=11, compaction requested=false 2024-12-01T14:55:21,523 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/info/695bc492c785402fa5a03f41534ca71f 2024-12-01T14:55:21,524 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/ns/c304f3e7733749de870693ee9a745391 2024-12-01T14:55:21,524 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/table/a362e02f68ca4932bbe9dcf0fbf2cd7b 2024-12-01T14:55:21,528 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-01T14:55:21,528 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T14:55:21,528 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T14:55:21,528 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733064921450Running coprocessor pre-close hooks at 1733064921450Disabling compacts and flushes for region at 1733064921450Disabling writes for close at 1733064921450Obtaining lock to block concurrent updates at 1733064921451 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733064921451Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1733064921452 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733064921452Flushing 1588230740/info: creating writer at 1733064921452Flushing 1588230740/info: appending metadata at 1733064921471 (+19 ms)Flushing 1588230740/info: closing flushed file at 1733064921471Flushing 1588230740/ns: creating writer at 1733064921474 (+3 ms)Flushing 1588230740/ns: appending metadata at 1733064921489 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733064921490 (+1 ms)Flushing 1588230740/table: creating writer at 1733064921493 (+3 ms)Flushing 1588230740/table: appending metadata at 1733064921510 (+17 ms)Flushing 1588230740/table: closing flushed file at 1733064921510Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21478d28: reopening flushed file at 1733064921513 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21038758: reopening flushed file at 1733064921517 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@690696f4: reopening flushed file at 1733064921520 (+3 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 72ms, sequenceid=11, compaction requested=false at 1733064921523 (+3 ms)Writing region close event to WAL at 1733064921525 (+2 ms)Running coprocessor post-close hooks at 1733064921528 (+3 ms)Closed at 1733064921528 2024-12-01T14:55:21,529 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-01T14:55:21,650 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.HRegionServer(976): stopping server 519b220a2ba1,40943,1733064918444; all regions closed. 2024-12-01T14:55:21,653 DEBUG [RS:1;519b220a2ba1:40943 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/oldWALs 2024-12-01T14:55:21,653 INFO [RS:1;519b220a2ba1:40943 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 519b220a2ba1%2C40943%2C1733064918444.meta:.meta(num 1733064919508) 2024-12-01T14:55:21,655 DEBUG [RS:1;519b220a2ba1:40943 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/oldWALs 2024-12-01T14:55:21,655 INFO [RS:1;519b220a2ba1:40943 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 519b220a2ba1%2C40943%2C1733064918444:(num 1733064919110) 2024-12-01T14:55:21,655 DEBUG [RS:1;519b220a2ba1:40943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:55:21,655 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T14:55:21,656 INFO [RS:1;519b220a2ba1:40943 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T14:55:21,656 INFO [RS:1;519b220a2ba1:40943 {}] hbase.ChoreService(370): Chore service for: regionserver/519b220a2ba1:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-01T14:55:21,656 INFO [RS:1;519b220a2ba1:40943 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T14:55:21,656 INFO [regionserver/519b220a2ba1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T14:55:21,657 INFO [RS:1;519b220a2ba1:40943 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40943 2024-12-01T14:55:21,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40943-0x10192ad978f0002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/519b220a2ba1,40943,1733064918444 2024-12-01T14:55:21,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T14:55:21,680 INFO [RS:1;519b220a2ba1:40943 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T14:55:21,688 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [519b220a2ba1,40943,1733064918444] 2024-12-01T14:55:21,696 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/519b220a2ba1,40943,1733064918444 already deleted, retry=false 2024-12-01T14:55:21,696 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 519b220a2ba1,40943,1733064918444 on 519b220a2ba1,42737,1733064918246 2024-12-01T14:55:21,701 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 519b220a2ba1,40943,1733064918444, splitWal=true, meta=true 2024-12-01T14:55:21,705 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 519b220a2ba1,40943,1733064918444 (carryingMeta=true) 519b220a2ba1,40943,1733064918444/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@54d03bcf[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-01T14:55:21,705 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 519b220a2ba1,40943,1733064918444, splitWal=true, meta=true 2024-12-01T14:55:21,706 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 519b220a2ba1,40943,1733064918444, splitWal=true, meta=true, isMeta: true 2024-12-01T14:55:21,707 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,40943,1733064918444-splitting 2024-12-01T14:55:21,708 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,40943,1733064918444-splitting dir is empty, no logs to split. 2024-12-01T14:55:21,708 INFO [PEWorker-1 {}] master.SplitWALManager(105): 519b220a2ba1,40943,1733064918444 WAL count=0, meta=true 2024-12-01T14:55:21,710 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,40943,1733064918444-splitting dir is empty, no logs to split. 2024-12-01T14:55:21,710 INFO [PEWorker-1 {}] master.SplitWALManager(105): 519b220a2ba1,40943,1733064918444 WAL count=0, meta=true 2024-12-01T14:55:21,710 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 519b220a2ba1,40943,1733064918444 WAL splitting is done? wals=0, meta=true 2024-12-01T14:55:21,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-01T14:55:21,713 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T14:55:21,714 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-01T14:55:21,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40943-0x10192ad978f0002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:55:21,788 INFO [RS:1;519b220a2ba1:40943 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T14:55:21,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40943-0x10192ad978f0002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:55:21,788 INFO [RS:1;519b220a2ba1:40943 {}] regionserver.HRegionServer(1031): Exiting; stopping=519b220a2ba1,40943,1733064918444; zookeeper connection closed. 2024-12-01T14:55:21,789 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@113b33a8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@113b33a8 2024-12-01T14:55:21,864 DEBUG [519b220a2ba1:42737 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-01T14:55:21,864 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=519b220a2ba1,34279,1733064918408 2024-12-01T14:55:21,865 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 519b220a2ba1,34279,1733064918408, state=OPENING 2024-12-01T14:55:21,874 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/519b220a2ba1:0 server-side Connection retries=45 2024-12-01T14:55:21,874 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:21,874 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:21,874 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T14:55:21,875 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:55:21,875 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T14:55:21,875 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T14:55:21,875 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T14:55:21,875 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40323 2024-12-01T14:55:21,877 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40323 connecting to ZooKeeper ensemble=127.0.0.1:58811 2024-12-01T14:55:21,877 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:21,877 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:21,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T14:55:21,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T14:55:21,896 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T14:55:21,896 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:55:21,896 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:55:21,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=519b220a2ba1,34279,1733064918408}] 2024-12-01T14:55:21,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:403230x0, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T14:55:21,905 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:403230x0, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-01T14:55:21,905 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-01T14:55:21,905 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40323-0x10192ad978f0003 connected 2024-12-01T14:55:21,906 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T14:55:21,906 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1/bucket.cache 2024-12-01T14:55:21,907 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-01T14:55:21,907 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-01T14:55:21,907 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-01T14:55:21,907 INFO [Thread-207 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-01T14:55:21,907 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-01T14:55:21,907 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T14:55:21,908 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:40323-0x10192ad978f0003, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-01T14:55:21,908 INFO [Thread-207 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-01T14:55:21,910 INFO [Thread-207 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-12-01T14:55:21,910 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40323-0x10192ad978f0003, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T14:55:21,910 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40323 2024-12-01T14:55:21,911 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40323 2024-12-01T14:55:21,911 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40323 2024-12-01T14:55:21,911 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40323 2024-12-01T14:55:21,911 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40323 2024-12-01T14:55:21,913 INFO [RS:2;519b220a2ba1:40323 {}] regionserver.HRegionServer(746): ClusterId : bcc9be81-f2fd-46ef-ab98-7c66ad66ea6c 2024-12-01T14:55:21,913 DEBUG [RS:2;519b220a2ba1:40323 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T14:55:21,915 DEBUG [Thread-207 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-01T14:55:21,915 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-01T14:55:21,915 INFO [Thread-207 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-12-01T14:55:21,915 INFO [Thread-207 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1/bucket.persistence is complete. 2024-12-01T14:55:21,922 DEBUG [RS:2;519b220a2ba1:40323 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T14:55:21,922 DEBUG [RS:2;519b220a2ba1:40323 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T14:55:21,931 DEBUG [RS:2;519b220a2ba1:40323 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T14:55:21,931 DEBUG [RS:2;519b220a2ba1:40323 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3624ec78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=519b220a2ba1/172.17.0.2:0 2024-12-01T14:55:21,942 DEBUG [RS:2;519b220a2ba1:40323 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;519b220a2ba1:40323 2024-12-01T14:55:21,942 INFO [RS:2;519b220a2ba1:40323 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T14:55:21,942 INFO [RS:2;519b220a2ba1:40323 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T14:55:21,942 DEBUG [RS:2;519b220a2ba1:40323 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T14:55:21,943 INFO [RS:2;519b220a2ba1:40323 {}] regionserver.HRegionServer(2659): reportForDuty to master=519b220a2ba1,42737,1733064918246 with port=40323, startcode=1733064921874 2024-12-01T14:55:21,943 DEBUG [RS:2;519b220a2ba1:40323 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T14:55:21,945 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50111, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T14:55:21,946 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42737 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 519b220a2ba1,40323,1733064921874 2024-12-01T14:55:21,946 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42737 {}] master.ServerManager(517): Registering regionserver=519b220a2ba1,40323,1733064921874 2024-12-01T14:55:21,947 DEBUG [RS:2;519b220a2ba1:40323 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46 2024-12-01T14:55:21,947 DEBUG [RS:2;519b220a2ba1:40323 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-01T14:55:21,947 DEBUG [RS:2;519b220a2ba1:40323 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-01T14:55:21,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T14:55:21,960 DEBUG [RS:2;519b220a2ba1:40323 {}] zookeeper.ZKUtil(111): regionserver:40323-0x10192ad978f0003, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/519b220a2ba1,40323,1733064921874 2024-12-01T14:55:21,960 WARN [RS:2;519b220a2ba1:40323 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T14:55:21,960 INFO [RS:2;519b220a2ba1:40323 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:55:21,960 DEBUG [RS:2;519b220a2ba1:40323 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,40323,1733064921874 2024-12-01T14:55:21,960 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [519b220a2ba1,40323,1733064921874] 2024-12-01T14:55:21,965 INFO [RS:2;519b220a2ba1:40323 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T14:55:21,968 INFO [RS:2;519b220a2ba1:40323 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T14:55:21,968 INFO [RS:2;519b220a2ba1:40323 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T14:55:21,968 INFO [RS:2;519b220a2ba1:40323 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:21,968 INFO [RS:2;519b220a2ba1:40323 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T14:55:21,969 INFO [RS:2;519b220a2ba1:40323 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T14:55:21,969 INFO [RS:2;519b220a2ba1:40323 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:21,969 DEBUG [RS:2;519b220a2ba1:40323 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:21,969 DEBUG [RS:2;519b220a2ba1:40323 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:21,969 DEBUG [RS:2;519b220a2ba1:40323 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:21,969 DEBUG [RS:2;519b220a2ba1:40323 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:21,970 DEBUG [RS:2;519b220a2ba1:40323 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:21,970 DEBUG [RS:2;519b220a2ba1:40323 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/519b220a2ba1:0, corePoolSize=2, maxPoolSize=2 2024-12-01T14:55:21,970 DEBUG [RS:2;519b220a2ba1:40323 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:21,970 DEBUG [RS:2;519b220a2ba1:40323 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:21,970 DEBUG [RS:2;519b220a2ba1:40323 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:21,970 DEBUG [RS:2;519b220a2ba1:40323 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:21,970 DEBUG [RS:2;519b220a2ba1:40323 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:21,970 DEBUG [RS:2;519b220a2ba1:40323 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/519b220a2ba1:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:55:21,970 DEBUG [RS:2;519b220a2ba1:40323 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/519b220a2ba1:0, corePoolSize=3, maxPoolSize=3 2024-12-01T14:55:21,970 DEBUG [RS:2;519b220a2ba1:40323 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/519b220a2ba1:0, corePoolSize=3, maxPoolSize=3 2024-12-01T14:55:21,970 INFO [RS:2;519b220a2ba1:40323 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:21,970 INFO [RS:2;519b220a2ba1:40323 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:21,971 INFO [RS:2;519b220a2ba1:40323 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:21,971 INFO [RS:2;519b220a2ba1:40323 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:21,971 INFO [RS:2;519b220a2ba1:40323 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:21,971 INFO [RS:2;519b220a2ba1:40323 {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,40323,1733064921874-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T14:55:21,984 INFO [RS:2;519b220a2ba1:40323 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T14:55:21,984 INFO [RS:2;519b220a2ba1:40323 {}] hbase.ChoreService(168): Chore ScheduledChore name=519b220a2ba1,40323,1733064921874-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:21,985 INFO [RS:2;519b220a2ba1:40323 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:21,985 INFO [RS:2;519b220a2ba1:40323 {}] regionserver.Replication(171): 519b220a2ba1,40323,1733064921874 started 2024-12-01T14:55:21,998 INFO [RS:2;519b220a2ba1:40323 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:55:21,998 INFO [RS:2;519b220a2ba1:40323 {}] regionserver.HRegionServer(1482): Serving as 519b220a2ba1,40323,1733064921874, RpcServer on 519b220a2ba1/172.17.0.2:40323, sessionid=0x10192ad978f0003 2024-12-01T14:55:21,998 DEBUG [RS:2;519b220a2ba1:40323 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T14:55:21,998 DEBUG [RS:2;519b220a2ba1:40323 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 519b220a2ba1,40323,1733064921874 2024-12-01T14:55:21,998 DEBUG [RS:2;519b220a2ba1:40323 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '519b220a2ba1,40323,1733064921874' 2024-12-01T14:55:21,998 DEBUG [RS:2;519b220a2ba1:40323 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-01T14:55:21,999 DEBUG [RS:2;519b220a2ba1:40323 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-01T14:55:21,999 DEBUG [RS:2;519b220a2ba1:40323 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T14:55:22,000 DEBUG [RS:2;519b220a2ba1:40323 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T14:55:22,000 DEBUG [RS:2;519b220a2ba1:40323 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 519b220a2ba1,40323,1733064921874 2024-12-01T14:55:22,000 DEBUG [RS:2;519b220a2ba1:40323 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '519b220a2ba1,40323,1733064921874' 2024-12-01T14:55:22,000 DEBUG [RS:2;519b220a2ba1:40323 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-01T14:55:22,000 DEBUG [RS:2;519b220a2ba1:40323 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-01T14:55:22,001 DEBUG [RS:2;519b220a2ba1:40323 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T14:55:22,001 INFO [RS:2;519b220a2ba1:40323 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T14:55:22,001 INFO [RS:2;519b220a2ba1:40323 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T14:55:22,049 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T14:55:22,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58975, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T14:55:22,053 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-01T14:55:22,053 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:55:22,053 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-01T14:55:22,054 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=519b220a2ba1%2C34279%2C1733064918408.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,34279,1733064918408, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/oldWALs, maxLogs=55 2024-12-01T14:55:22,074 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,34279,1733064918408/519b220a2ba1%2C34279%2C1733064918408.meta.1733064922055.meta 2024-12-01T14:55:22,075 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-01T14:55:22,075 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:55:22,075 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-01T14:55:22,075 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-01T14:55:22,075 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-01T14:55:22,076 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-01T14:55:22,076 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:55:22,076 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-01T14:55:22,076 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-01T14:55:22,077 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-01T14:55:22,077 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T14:55:22,077 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:22,078 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/info/695bc492c785402fa5a03f41534ca71f, delay=942 ms 2024-12-01T14:55:22,079 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/info/695bc492c785402fa5a03f41534ca71f 2024-12-01T14:55:22,079 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:22,079 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-01T14:55:22,080 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T14:55:22,080 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:22,082 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/ns/c304f3e7733749de870693ee9a745391, delay=997 ms 2024-12-01T14:55:22,083 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/ns/c304f3e7733749de870693ee9a745391 2024-12-01T14:55:22,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:22,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-01T14:55:22,083 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T14:55:22,083 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:22,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:22,084 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-01T14:55:22,084 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T14:55:22,084 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:55:22,085 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/table/a362e02f68ca4932bbe9dcf0fbf2cd7b, delay=977 ms 2024-12-01T14:55:22,086 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/table/a362e02f68ca4932bbe9dcf0fbf2cd7b 2024-12-01T14:55:22,086 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:55:22,086 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T14:55:22,087 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740 2024-12-01T14:55:22,087 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740 2024-12-01T14:55:22,088 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T14:55:22,088 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T14:55:22,088 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T14:55:22,089 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T14:55:22,090 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69961259, jitterRate=0.042503997683525085}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T14:55:22,090 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-01T14:55:22,091 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733064922076Writing region info on filesystem at 1733064922076Initializing all the Stores at 1733064922076Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733064922076Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733064922076Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733064922076Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733064922076Cleaning up temporary data from old regions at 1733064922088 (+12 ms)Running coprocessor post-open hooks at 1733064922090 (+2 ms)Region opened successfully at 1733064922091 (+1 ms) 2024-12-01T14:55:22,092 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1733064922048 2024-12-01T14:55:22,095 DEBUG [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-01T14:55:22,095 INFO [RS_OPEN_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-01T14:55:22,095 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=519b220a2ba1,34279,1733064918408 2024-12-01T14:55:22,096 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 519b220a2ba1,34279,1733064918408, state=OPEN 2024-12-01T14:55:22,101 INFO [RS:2;519b220a2ba1:40323 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:55:22,102 INFO [RS:2;519b220a2ba1:40323 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=519b220a2ba1%2C40323%2C1733064921874, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,40323,1733064921874, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/oldWALs, maxLogs=55 2024-12-01T14:55:22,110 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42737 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=519b220a2ba1,34279,1733064918408, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-01T14:55:22,118 INFO [RS:2;519b220a2ba1:40323 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,40323,1733064921874/519b220a2ba1%2C40323%2C1733064921874.1733064922102 2024-12-01T14:55:22,118 DEBUG [RS:2;519b220a2ba1:40323 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-01T14:55:22,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T14:55:22,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T14:55:22,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40323-0x10192ad978f0003, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T14:55:22,121 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=519b220a2ba1,34279,1733064918408 2024-12-01T14:55:22,121 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:55:22,121 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:55:22,121 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:55:22,123 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-01T14:55:22,123 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=519b220a2ba1,34279,1733064918408 in 225 msec 2024-12-01T14:55:22,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-01T14:55:22,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 411 msec 2024-12-01T14:55:22,125 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(207): 519b220a2ba1,40943,1733064918444 had 2 regions 2024-12-01T14:55:22,126 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 519b220a2ba1,40943,1733064918444, splitWal=true, meta=true, isMeta: false 2024-12-01T14:55:22,126 INFO [PEWorker-4 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,40943,1733064918444-splitting dir is empty, no logs to split. 2024-12-01T14:55:22,126 INFO [PEWorker-4 {}] master.SplitWALManager(105): 519b220a2ba1,40943,1733064918444 WAL count=0, meta=false 2024-12-01T14:55:22,127 INFO [PEWorker-4 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/WALs/519b220a2ba1,40943,1733064918444-splitting dir is empty, no logs to split. 2024-12-01T14:55:22,127 INFO [PEWorker-4 {}] master.SplitWALManager(105): 519b220a2ba1,40943,1733064918444 WAL count=0, meta=false 2024-12-01T14:55:22,127 DEBUG [PEWorker-4 {}] procedure.ServerCrashProcedure(329): Check if 519b220a2ba1,40943,1733064918444 WAL splitting is done? wals=0, meta=false 2024-12-01T14:55:22,129 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733064919681, region=5754fb1c74471df5775e3ac5ebf5eb44, ASSIGN}] 2024-12-01T14:55:22,130 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733064919681, region=5754fb1c74471df5775e3ac5ebf5eb44, ASSIGN 2024-12-01T14:55:22,131 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733064919681, region=5754fb1c74471df5775e3ac5ebf5eb44, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-01T14:55:22,281 DEBUG [519b220a2ba1:42737 {}] balancer.BalancerClusterState(204): Hosts are {519b220a2ba1=0} racks are {/default-rack=0} 2024-12-01T14:55:22,282 DEBUG [519b220a2ba1:42737 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T14:55:22,282 DEBUG [519b220a2ba1:42737 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T14:55:22,282 DEBUG [519b220a2ba1:42737 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T14:55:22,282 DEBUG [519b220a2ba1:42737 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T14:55:22,282 INFO [519b220a2ba1:42737 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T14:55:22,282 INFO [519b220a2ba1:42737 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T14:55:22,282 DEBUG [519b220a2ba1:42737 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T14:55:22,282 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5754fb1c74471df5775e3ac5ebf5eb44, regionState=OPENING, regionLocation=519b220a2ba1,34279,1733064918408 2024-12-01T14:55:22,284 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 519b220a2ba1:40943 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 519b220a2ba1/172.17.0.2:40943 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:55:22,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=519b220a2ba1,40943,1733064918444, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=519b220a2ba1,40943,1733064918444, seqNum=-1, error=java.net.ConnectException: Call to address=519b220a2ba1:40943 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 519b220a2ba1/172.17.0.2:40943 2024-12-01T14:55:22,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=519b220a2ba1,40943,1733064918444, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-01T14:55:22,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=519b220a2ba1,40943,1733064918444, seqNum=-1 from cache 2024-12-01T14:55:22,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 519b220a2ba1:40943 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 519b220a2ba1/172.17.0.2:40943 2024-12-01T14:55:22,392 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T14:55:22,392 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=519b220a2ba1,34279,1733064918408, seqNum=-1] 2024-12-01T14:55:22,393 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T14:55:22,396 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50691, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T14:55:22,400 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733064919681, region=5754fb1c74471df5775e3ac5ebf5eb44, ASSIGN because future has completed 2024-12-01T14:55:22,401 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5754fb1c74471df5775e3ac5ebf5eb44, server=519b220a2ba1,34279,1733064918408}] 2024-12-01T14:55:22,498 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-01T14:55:22,499 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T14:55:22,499 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T14:55:22,499 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:55:22,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:55:22,500 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T14:55:22,500 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-01T14:55:22,500 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2026433345, stopped=false 2024-12-01T14:55:22,501 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=519b220a2ba1,42737,1733064918246 2024-12-01T14:55:22,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T14:55:22,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40323-0x10192ad978f0003, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T14:55:22,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T14:55:22,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40323-0x10192ad978f0003, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:22,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:22,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:22,547 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-01T14:55:22,547 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T14:55:22,548 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T14:55:22,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:55:22,548 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40323-0x10192ad978f0003, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T14:55:22,548 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '519b220a2ba1,34279,1733064918408' ***** 2024-12-01T14:55:22,549 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T14:55:22,549 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-01T14:55:22,549 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '519b220a2ba1,40323,1733064921874' ***** 2024-12-01T14:55:22,549 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-01T14:55:22,549 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T14:55:22,549 INFO [RS:0;519b220a2ba1:34279 {}] bucket.BucketCache(1738): Disabling cache 2024-12-01T14:55:22,550 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-01T14:55:22,550 INFO [RS:0;519b220a2ba1:34279 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1/bucket.persistence 2024-12-01T14:55:22,550 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-01T14:55:22,550 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-01T14:55:22,550 INFO [RS:0;519b220a2ba1:34279 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-01T14:55:22,550 INFO [RS:2;519b220a2ba1:40323 {}] bucket.BucketCache(1738): Disabling cache 2024-12-01T14:55:22,551 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-01T14:55:22,550 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-01T14:55:22,551 INFO [RS:2;519b220a2ba1:40323 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b5a5ed18-186a-588e-c3cf-47c07c3af0e1/bucket.persistence 2024-12-01T14:55:22,551 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-01T14:55:22,551 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-01T14:55:22,551 INFO [RS:2;519b220a2ba1:40323 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-01T14:55:22,551 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-01T14:55:22,554 DEBUG [RSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(271): Request to 519b220a2ba1,34279,1733064918408 failed, try=0 org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=519b220a2ba1:34279 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$Stub.executeProcedures(AdminProtos.java:35059) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.lambda$executeProcedures$20(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:103) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:121) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.executeProcedures(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 11 more 2024-12-01T14:55:22,556 WARN [RSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(288): 519b220a2ba1,34279,1733064918408 tells us DoNotRetry due to org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=519b220a2ba1:34279 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException, try=0, give up 2024-12-01T14:55:22,556 WARN [RSProcedureDispatcher-pool-0 {}] assignment.RegionRemoteProcedureBase(139): The remote operation pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5754fb1c74471df5775e3ac5ebf5eb44, server=519b220a2ba1,34279,1733064918408 for region state=OPENING, location=519b220a2ba1,34279,1733064918408, table=testBlockEvictionOnGracefulStop1733064919681, region=5754fb1c74471df5775e3ac5ebf5eb44 to server 519b220a2ba1,34279,1733064918408 failed org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=519b220a2ba1:34279 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$Stub.executeProcedures(AdminProtos.java:35059) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.lambda$executeProcedures$20(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:103) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:121) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.executeProcedures(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 11 more 2024-12-01T14:55:22,560 DEBUG [RS:0;519b220a2ba1:34279 {}] bucket.BucketCache(1487): Thread RS:0;519b220a2ba1:34279 started persisting bucket cache to file 2024-12-01T14:55:22,560 DEBUG [RS:0;519b220a2ba1:34279 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-01T14:55:22,561 DEBUG [RS:0;519b220a2ba1:34279 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-01T14:55:22,561 DEBUG [RS:2;519b220a2ba1:40323 {}] bucket.BucketCache(1487): Thread RS:2;519b220a2ba1:40323 started persisting bucket cache to file 2024-12-01T14:55:22,561 DEBUG [RS:2;519b220a2ba1:40323 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-01T14:55:22,561 DEBUG [RS:2;519b220a2ba1:40323 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-01T14:55:22,563 DEBUG [RS:0;519b220a2ba1:34279 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-01T14:55:22,563 DEBUG [RS:0;519b220a2ba1:34279 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733064922560 2024-12-01T14:55:22,563 DEBUG [RS:0;519b220a2ba1:34279 {}] bucket.BucketCache(1509): Thread RS:0;519b220a2ba1:34279 finished persisting bucket cache to file, renaming 2024-12-01T14:55:22,563 INFO [RS:0;519b220a2ba1:34279 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T14:55:22,564 INFO [RS:0;519b220a2ba1:34279 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-01T14:55:22,564 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T14:55:22,564 INFO [RS:0;519b220a2ba1:34279 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-01T14:55:22,564 DEBUG [RS:2;519b220a2ba1:40323 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-01T14:55:22,564 INFO [RS:0;519b220a2ba1:34279 {}] regionserver.HRegionServer(959): stopping server 519b220a2ba1,34279,1733064918408 2024-12-01T14:55:22,564 DEBUG [RS:2;519b220a2ba1:40323 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733064922561 2024-12-01T14:55:22,564 INFO [RS:0;519b220a2ba1:34279 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T14:55:22,564 DEBUG [RS:2;519b220a2ba1:40323 {}] bucket.BucketCache(1509): Thread RS:2;519b220a2ba1:40323 finished persisting bucket cache to file, renaming 2024-12-01T14:55:22,564 INFO [RS:0;519b220a2ba1:34279 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;519b220a2ba1:34279. 2024-12-01T14:55:22,564 DEBUG [RS:0;519b220a2ba1:34279 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T14:55:22,564 DEBUG [RS:0;519b220a2ba1:34279 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:55:22,564 INFO [RS:2;519b220a2ba1:40323 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T14:55:22,564 INFO [RS:2;519b220a2ba1:40323 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-01T14:55:22,564 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T14:55:22,564 INFO [RS:2;519b220a2ba1:40323 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-01T14:55:22,564 INFO [RS:0;519b220a2ba1:34279 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T14:55:22,564 INFO [RS:2;519b220a2ba1:40323 {}] regionserver.HRegionServer(959): stopping server 519b220a2ba1,40323,1733064921874 2024-12-01T14:55:22,564 INFO [RS:0;519b220a2ba1:34279 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T14:55:22,564 INFO [RS:2;519b220a2ba1:40323 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T14:55:22,564 INFO [RS:0;519b220a2ba1:34279 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T14:55:22,564 INFO [RS:2;519b220a2ba1:40323 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;519b220a2ba1:40323. 2024-12-01T14:55:22,564 INFO [RS:0;519b220a2ba1:34279 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-01T14:55:22,564 DEBUG [RS:2;519b220a2ba1:40323 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T14:55:22,564 DEBUG [RS:2;519b220a2ba1:40323 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:55:22,565 INFO [RS:2;519b220a2ba1:40323 {}] regionserver.HRegionServer(976): stopping server 519b220a2ba1,40323,1733064921874; all regions closed. 2024-12-01T14:55:22,565 INFO [RS:0;519b220a2ba1:34279 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-01T14:55:22,565 DEBUG [RS:0;519b220a2ba1:34279 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-01T14:55:22,565 DEBUG [RS:0;519b220a2ba1:34279 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-01T14:55:22,565 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-01T14:55:22,565 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T14:55:22,565 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T14:55:22,565 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T14:55:22,565 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T14:55:22,565 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T14:55:22,565 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=485 B heapSize=1.70 KB 2024-12-01T14:55:22,566 DEBUG [RS:2;519b220a2ba1:40323 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/oldWALs 2024-12-01T14:55:22,566 INFO [RS:2;519b220a2ba1:40323 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 519b220a2ba1%2C40323%2C1733064921874:(num 1733064922102) 2024-12-01T14:55:22,566 DEBUG [RS:2;519b220a2ba1:40323 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:55:22,566 INFO [RS:2;519b220a2ba1:40323 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T14:55:22,566 INFO [RS:2;519b220a2ba1:40323 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T14:55:22,567 INFO [RS:2;519b220a2ba1:40323 {}] hbase.ChoreService(370): Chore service for: regionserver/519b220a2ba1:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-01T14:55:22,567 INFO [RS:2;519b220a2ba1:40323 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T14:55:22,567 INFO [regionserver/519b220a2ba1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T14:55:22,567 INFO [RS:2;519b220a2ba1:40323 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T14:55:22,567 INFO [RS:2;519b220a2ba1:40323 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T14:55:22,567 INFO [RS:2;519b220a2ba1:40323 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T14:55:22,567 INFO [RS:2;519b220a2ba1:40323 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40323 2024-12-01T14:55:22,572 INFO [regionserver/519b220a2ba1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T14:55:22,575 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d05c621b18c042fb9808d9de406fe3bd 2024-12-01T14:55:22,578 INFO [regionserver/519b220a2ba1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T14:55:22,589 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/info/d05c621b18c042fb9808d9de406fe3bd is 209, key is testBlockEvictionOnGracefulStop1733064919681,,1733064919685.5754fb1c74471df5775e3ac5ebf5eb44./info:regioninfo/1733064922282/Put/seqid=0 2024-12-01T14:55:22,589 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=485 B at sequenceid=18 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/info/d05c621b18c042fb9808d9de406fe3bd 2024-12-01T14:55:22,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40323-0x10192ad978f0003, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/519b220a2ba1,40323,1733064921874 2024-12-01T14:55:22,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T14:55:22,590 INFO [RS:2;519b220a2ba1:40323 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T14:55:22,592 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/.tmp/info/d05c621b18c042fb9808d9de406fe3bd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/info/d05c621b18c042fb9808d9de406fe3bd 2024-12-01T14:55:22,593 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/info/d05c621b18c042fb9808d9de406fe3bd, delay=958 ms 2024-12-01T14:55:22,593 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/info/d05c621b18c042fb9808d9de406fe3bd, entries=3, sequenceid=18, filesize=5.9 K 2024-12-01T14:55:22,594 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~485 B/485, heapSize ~952 B/952, currentSize=0 B/0 for 1588230740 in 29ms, sequenceid=18, compaction requested=false 2024-12-01T14:55:22,595 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/info/695bc492c785402fa5a03f41534ca71f 2024-12-01T14:55:22,595 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/info/d05c621b18c042fb9808d9de406fe3bd 2024-12-01T14:55:22,595 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/ns/c304f3e7733749de870693ee9a745391 2024-12-01T14:55:22,596 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/table/a362e02f68ca4932bbe9dcf0fbf2cd7b 2024-12-01T14:55:22,600 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/data/hbase/meta/1588230740/recovered.edits/21.seqid, newMaxSeqId=21, maxSeqId=14 2024-12-01T14:55:22,600 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T14:55:22,600 INFO [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T14:55:22,600 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733064922565Running coprocessor pre-close hooks at 1733064922565Disabling compacts and flushes for region at 1733064922565Disabling writes for close at 1733064922565Obtaining lock to block concurrent updates at 1733064922565Preparing flush snapshotting stores in 1588230740 at 1733064922565Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=485, getHeapSize=1672, getOffHeapSize=0, getCellsCount=3 at 1733064922566 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733064922566Flushing 1588230740/info: creating writer at 1733064922566Flushing 1588230740/info: appending metadata at 1733064922588 (+22 ms)Flushing 1588230740/info: closing flushed file at 1733064922588Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7421cfe1: reopening flushed file at 1733064922592 (+4 ms)Finished flush of dataSize ~485 B/485, heapSize ~952 B/952, currentSize=0 B/0 for 1588230740 in 29ms, sequenceid=18, compaction requested=false at 1733064922594 (+2 ms)Writing region close event to WAL at 1733064922596 (+2 ms)Running coprocessor post-close hooks at 1733064922600 (+4 ms)Closed at 1733064922600 2024-12-01T14:55:22,600 DEBUG [RS_CLOSE_META-regionserver/519b220a2ba1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-01T14:55:22,604 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [519b220a2ba1,40323,1733064921874] 2024-12-01T14:55:22,612 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/519b220a2ba1,40323,1733064921874 already deleted, retry=false 2024-12-01T14:55:22,613 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 519b220a2ba1,40323,1733064921874 expired; onlineServers=1 2024-12-01T14:55:22,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40323-0x10192ad978f0003, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:55:22,705 INFO [RS:2;519b220a2ba1:40323 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T14:55:22,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40323-0x10192ad978f0003, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:55:22,705 INFO [RS:2;519b220a2ba1:40323 {}] regionserver.HRegionServer(1031): Exiting; stopping=519b220a2ba1,40323,1733064921874; zookeeper connection closed. 2024-12-01T14:55:22,705 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@20f0fa1c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@20f0fa1c 2024-12-01T14:55:22,765 INFO [RS:0;519b220a2ba1:34279 {}] regionserver.HRegionServer(976): stopping server 519b220a2ba1,34279,1733064918408; all regions closed. 2024-12-01T14:55:22,771 DEBUG [RS:0;519b220a2ba1:34279 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/oldWALs 2024-12-01T14:55:22,771 INFO [RS:0;519b220a2ba1:34279 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 519b220a2ba1%2C34279%2C1733064918408.meta:.meta(num 1733064922055) 2024-12-01T14:55:22,775 DEBUG [RS:0;519b220a2ba1:34279 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/oldWALs 2024-12-01T14:55:22,775 INFO [RS:0;519b220a2ba1:34279 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 519b220a2ba1%2C34279%2C1733064918408:(num 1733064919110) 2024-12-01T14:55:22,775 DEBUG [RS:0;519b220a2ba1:34279 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:55:22,775 INFO [RS:0;519b220a2ba1:34279 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T14:55:22,775 INFO [RS:0;519b220a2ba1:34279 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T14:55:22,775 INFO [RS:0;519b220a2ba1:34279 {}] hbase.ChoreService(370): Chore service for: regionserver/519b220a2ba1:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-01T14:55:22,775 INFO [RS:0;519b220a2ba1:34279 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T14:55:22,775 INFO [regionserver/519b220a2ba1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T14:55:22,775 INFO [RS:0;519b220a2ba1:34279 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34279 2024-12-01T14:55:22,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/519b220a2ba1,34279,1733064918408 2024-12-01T14:55:22,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T14:55:22,793 INFO [RS:0;519b220a2ba1:34279 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T14:55:22,805 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [519b220a2ba1,34279,1733064918408] 2024-12-01T14:55:22,813 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/519b220a2ba1,34279,1733064918408 already deleted, retry=false 2024-12-01T14:55:22,813 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 519b220a2ba1,34279,1733064918408 expired; onlineServers=0 2024-12-01T14:55:22,813 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '519b220a2ba1,42737,1733064918246' ***** 2024-12-01T14:55:22,813 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-01T14:55:22,813 INFO [M:0;519b220a2ba1:42737 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T14:55:22,814 INFO [M:0;519b220a2ba1:42737 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T14:55:22,814 DEBUG [M:0;519b220a2ba1:42737 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-01T14:55:22,814 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-01T14:55:22,814 DEBUG [M:0;519b220a2ba1:42737 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-01T14:55:22,814 DEBUG [master/519b220a2ba1:0:becomeActiveMaster-HFileCleaner.small.0-1733064918848 {}] cleaner.HFileCleaner(306): Exit Thread[master/519b220a2ba1:0:becomeActiveMaster-HFileCleaner.small.0-1733064918848,5,FailOnTimeoutGroup] 2024-12-01T14:55:22,814 DEBUG [master/519b220a2ba1:0:becomeActiveMaster-HFileCleaner.large.0-1733064918847 {}] cleaner.HFileCleaner(306): Exit Thread[master/519b220a2ba1:0:becomeActiveMaster-HFileCleaner.large.0-1733064918847,5,FailOnTimeoutGroup] 2024-12-01T14:55:22,815 INFO [M:0;519b220a2ba1:42737 {}] hbase.ChoreService(370): Chore service for: master/519b220a2ba1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-01T14:55:22,816 INFO [M:0;519b220a2ba1:42737 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T14:55:22,816 DEBUG [M:0;519b220a2ba1:42737 {}] master.HMaster(1795): Stopping service threads 2024-12-01T14:55:22,816 INFO [M:0;519b220a2ba1:42737 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-01T14:55:22,816 INFO [M:0;519b220a2ba1:42737 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-01T14:55:22,817 INFO [M:0;519b220a2ba1:42737 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-01T14:55:22,817 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-01T14:55:22,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-01T14:55:22,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:55:22,827 DEBUG [M:0;519b220a2ba1:42737 {}] zookeeper.ZKUtil(347): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-01T14:55:22,827 WARN [M:0;519b220a2ba1:42737 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-01T14:55:22,827 INFO [M:0;519b220a2ba1:42737 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/.lastflushedseqids 2024-12-01T14:55:22,833 INFO [M:0;519b220a2ba1:42737 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-01T14:55:22,833 INFO [M:0;519b220a2ba1:42737 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-01T14:55:22,834 DEBUG [M:0;519b220a2ba1:42737 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-01T14:55:22,834 INFO [M:0;519b220a2ba1:42737 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:55:22,834 DEBUG [M:0;519b220a2ba1:42737 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:55:22,834 DEBUG [M:0;519b220a2ba1:42737 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-01T14:55:22,834 DEBUG [M:0;519b220a2ba1:42737 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:55:22,834 INFO [M:0;519b220a2ba1:42737 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.94 KB heapSize=44.16 KB 2024-12-01T14:55:22,843 DEBUG [M:0;519b220a2ba1:42737 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1f7ad543d70543968a5d83c3b196bc75 2024-12-01T14:55:22,856 DEBUG [M:0;519b220a2ba1:42737 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1f7ad543d70543968a5d83c3b196bc75 is 82, key is hbase:meta,,1/info:regioninfo/1733064922095/Put/seqid=0 2024-12-01T14:55:22,856 INFO [M:0;519b220a2ba1:42737 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1f7ad543d70543968a5d83c3b196bc75 2024-12-01T14:55:22,863 DEBUG [M:0;519b220a2ba1:42737 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1e51e25af4f74dc4992a44963ada1af0 2024-12-01T14:55:22,873 DEBUG [M:0;519b220a2ba1:42737 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e51e25af4f74dc4992a44963ada1af0 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733064920054/Put/seqid=0 2024-12-01T14:55:22,874 INFO [M:0;519b220a2ba1:42737 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.71 KB at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e51e25af4f74dc4992a44963ada1af0 2024-12-01T14:55:22,880 DEBUG [M:0;519b220a2ba1:42737 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9afc3b8103f5454cba397d832c1f7497 2024-12-01T14:55:22,891 DEBUG [M:0;519b220a2ba1:42737 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9afc3b8103f5454cba397d832c1f7497 is 69, key is 519b220a2ba1,34279,1733064918408/rs:state/1733064918914/Put/seqid=0 2024-12-01T14:55:22,892 INFO [M:0;519b220a2ba1:42737 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9afc3b8103f5454cba397d832c1f7497 2024-12-01T14:55:22,893 INFO [M:0;519b220a2ba1:42737 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9afc3b8103f5454cba397d832c1f7497 2024-12-01T14:55:22,894 DEBUG [M:0;519b220a2ba1:42737 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1f7ad543d70543968a5d83c3b196bc75 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1f7ad543d70543968a5d83c3b196bc75 2024-12-01T14:55:22,895 INFO [M:0;519b220a2ba1:42737 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1f7ad543d70543968a5d83c3b196bc75, entries=14, sequenceid=90, filesize=5.9 K 2024-12-01T14:55:22,895 DEBUG [M:0;519b220a2ba1:42737 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e51e25af4f74dc4992a44963ada1af0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1e51e25af4f74dc4992a44963ada1af0 2024-12-01T14:55:22,896 INFO [M:0;519b220a2ba1:42737 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1e51e25af4f74dc4992a44963ada1af0, entries=11, sequenceid=90, filesize=7.8 K 2024-12-01T14:55:22,897 DEBUG [M:0;519b220a2ba1:42737 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9afc3b8103f5454cba397d832c1f7497 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9afc3b8103f5454cba397d832c1f7497 2024-12-01T14:55:22,898 INFO [M:0;519b220a2ba1:42737 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9afc3b8103f5454cba397d832c1f7497 2024-12-01T14:55:22,898 INFO [M:0;519b220a2ba1:42737 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4d123e28-d9bd-94b7-378c-d1cc641c1965/2b554275-eb03-47a8-eadb-41766f183a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9afc3b8103f5454cba397d832c1f7497, entries=3, sequenceid=90, filesize=5.3 K 2024-12-01T14:55:22,899 INFO [M:0;519b220a2ba1:42737 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.94 KB/35780, heapSize ~43.87 KB/44920, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 65ms, sequenceid=90, compaction requested=false 2024-12-01T14:55:22,900 INFO [M:0;519b220a2ba1:42737 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:55:22,901 DEBUG [M:0;519b220a2ba1:42737 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733064922834Disabling compacts and flushes for region at 1733064922834Disabling writes for close at 1733064922834Obtaining lock to block concurrent updates at 1733064922834Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733064922834Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35780, getHeapSize=45160, getOffHeapSize=0, getCellsCount=111 at 1733064922835 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733064922835Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733064922835Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733064922855 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733064922856 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733064922858 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733064922873 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733064922873Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733064922875 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733064922891 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733064922891Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cc34bc5: reopening flushed file at 1733064922894 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5440c8: reopening flushed file at 1733064922895 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34599def: reopening flushed file at 1733064922897 (+2 ms)Finished flush of dataSize ~34.94 KB/35780, heapSize ~43.87 KB/44920, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 65ms, sequenceid=90, compaction requested=false at 1733064922899 (+2 ms)Writing region close event to WAL at 1733064922900 (+1 ms)Closed at 1733064922900 2024-12-01T14:55:22,901 INFO [M:0;519b220a2ba1:42737 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-01T14:55:22,901 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T14:55:22,901 INFO [M:0;519b220a2ba1:42737 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42737 2024-12-01T14:55:22,902 INFO [M:0;519b220a2ba1:42737 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T14:55:22,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:55:22,905 INFO [RS:0;519b220a2ba1:34279 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T14:55:22,905 INFO [RS:0;519b220a2ba1:34279 {}] regionserver.HRegionServer(1031): Exiting; stopping=519b220a2ba1,34279,1733064918408; zookeeper connection closed. 2024-12-01T14:55:22,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34279-0x10192ad978f0001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:55:22,905 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@26cb5876 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@26cb5876 2024-12-01T14:55:22,905 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-01T14:55:23,021 INFO [M:0;519b220a2ba1:42737 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T14:55:23,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:55:23,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42737-0x10192ad978f0000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:55:23,027 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-01T14:55:23,028 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-01T14:55:23,028 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:55:23,037 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=427 (was 447), ProcessCount=11 (was 11), AvailableMemoryMB=5075 (was 5150)